Rt updates (#303)

* feat(reports):Adds RT completeness charts

* fix(reports): grab tooltip values from jinja html

* fix(reports): corrected dumb linting errors

README.md

@@ -214,4 +214,4 @@ Open a terminal and navigate to the root folder of a locally cloned repo and ent
 docker-compose run --rm --service-ports calitp_reports /bin/bash
 ```
 
-If google credentials are already configured on the host, the local credential files should already be mounted in the container, but it may only be necessary to run `gcloud auth application-default login` from within the container.
+If google credentials are already configured on the host, the local credential files should already be mounted in the container, but it may only be necessary to run `gcloud auth application-default login` from within the container.

reports/generate_reports_data.py

@@ -6,8 +6,9 @@
 from typing import Optional
 
 import numpy as np
+import pandas as pd
 import typer
-from calitp_data_analysis.sql import get_engine  # type: ignore
+from calitp_data_analysis.sql import get_engine, query_sql  # type: ignore
 from siuba import _, arrange, collect  # type: ignore
 from siuba import filter as filtr  # type: ignore
 from siuba import left_join, mutate, pipe, rename, select, spread  # type: ignore
@@ -133,6 +134,113 @@ def generate_daily_service_hours(itp_id: int, date_start, date_end):
     )
 
 
+@cache
+def _rt_completeness():
+    return query_sql(
+        """
+            SELECT
+                    organization_itp_id as calitp_itp_id,
+                    DATETIME(service_date) as service_date,
+                    percent_of_trips_with_TU_messages, percent_of_trips_with_VP_messages
+                    FROM `cal-itp-data-infra.mart_gtfs_quality.fct_daily_trip_updates_vehicle_positions_completeness`
+                     """,
+        as_df=True,
+    )
+
+
+def generate_rt_completeness(itp_id: int, date_start: str, date_end: str):
+    df = _rt_completeness()
+    date_start = pd.to_datetime(date_start)
+    date_end = pd.to_datetime(date_end)
+
+    # Filter the DataFrame based on the conditions
+    return df[
+        (df["calitp_itp_id"] == itp_id)
+        & (df["service_date"] >= date_start)
+        & (df["service_date"] <= date_end)
+    ]
+
+
+@cache
+def _median_tu_age():
+    return query_sql(
+        """
+SELECT
+  -- mas.base64_url, #Issues are here, this is producing orgs without proper cal-itp ids, they don't exist down the chain
+  organization_name,
+  organization_itp_id as calitp_itp_id,
+DATETIME(DATE_TRUNC(mas.dt, DAY)) as service_date,
+  AVG(mas.median_trip_update_message_age) AS avg_median_trip_update_message_age
+
+FROM `mart_gtfs_quality.fct_daily_trip_updates_message_age_summary` mas
+
+LEFT JOIN `mart_transit_database.dim_gtfs_datasets` dgd
+  ON mas.base64_url = dgd.base64_url
+LEFT JOIN `mart_transit_database.dim_provider_gtfs_data` dpgd
+  ON dgd.key = dpgd.trip_updates_gtfs_dataset_key
+
+WHERE mas.dt < DATE_TRUNC(CURRENT_DATE('America/Los_Angeles'), DAY)
+AND organization_itp_id is not null
+
+GROUP BY 1, 2, 3
+""",
+        as_df=True,
+    )
+
+
+def generate_ave_median_tu_age(itp_id: int, date_start, date_end):
+    df = _median_tu_age()
+    date_start = pd.to_datetime(date_start)
+    date_end = pd.to_datetime(date_end)
+
+    # Filter the DataFrame based on the conditions
+    return df[
+        (df["calitp_itp_id"] == itp_id)
+        & (df["service_date"] >= date_start)
+        & (df["service_date"] <= date_end)
+    ]
+
+
+@cache
+def _median_vp_age():
+    return query_sql(
+        """
+SELECT
+  -- mas.base64_url, #Issues are here, this is producing orgs without proper cal-itp ids
+  organization_name,
+  organization_itp_id as calitp_itp_id,
+  DATETIME(DATE_TRUNC(mas.dt, DAY)) AS service_date,
+  AVG(mas.median_vehicle_message_age) AS avg_median_vehicle_message_age
+
+FROM `mart_gtfs_quality.fct_daily_vehicle_positions_message_age_summary` mas
+
+LEFT JOIN `mart_transit_database.dim_gtfs_datasets` dgd
+  ON mas.base64_url = dgd.base64_url
+LEFT JOIN `mart_transit_database.dim_provider_gtfs_data` dpgd
+  ON dgd.key = dpgd.vehicle_positions_gtfs_dataset_key
+
+WHERE mas.dt < DATE_TRUNC(CURRENT_DATE('America/Los_Angeles'), DAY)
+AND organization_itp_id is not null
+
+GROUP BY 1, 2, 3
+""",
+        as_df=True,
+    )
+
+
+def generate_ave_median_vp_age(itp_id: int, date_start, date_end):
+    df = _median_vp_age()
+    date_start = pd.to_datetime(date_start)
+    date_end = pd.to_datetime(date_end)
+
+    # Filter the DataFrame based on the conditions
+    return df[
+        (df["calitp_itp_id"] == itp_id)
+        & (df["service_date"] >= date_start)
+        & (df["service_date"] <= date_end)
+    ]
+
+
 @cache
 def _guideline_check():
     return (
@@ -288,6 +396,24 @@ def dump_report_data(
     service_hours = generate_daily_service_hours(itp_id, date_start, date_end)
     service_hours.to_json(out_dir / "2_daily_service_hours.json", orient="records")
 
+    # 2_gtfs_rt_completeness.json
+    if verbose:
+        print(f"Generating rt_complete for {itp_id}")
+    rt_complete = generate_rt_completeness(itp_id, date_start, date_end)
+    rt_complete.to_json(out_dir / "2_gtfs_rt_completeness.json", orient="records")
+
+    # 2_tu_message_age.json
+    if verbose:
+        print(f"Generating tu_age for {itp_id}")
+    ave_median_tu_age = generate_ave_median_tu_age(itp_id, date_start, date_end)
+    ave_median_tu_age.to_json(out_dir / "2_ave_median_tu_age.json", orient="records")
+
+    # 2_vp_message_age.json
+    if verbose:
+        print(f"Generating vp_age for {itp_id}")
+    ave_median_vp_age = generate_ave_median_vp_age(itp_id, date_start, date_end)
+    ave_median_vp_age.to_json(out_dir / "2_ave_median_vp_age.json", orient="records")
+
     # 3_stops_changed.json
     if verbose:
         print(f"Generating stops changed for {itp_id}")

reports/test_emails.csv

@@ -1,4 +1,4 @@
 First Name,Last Name,Email,Company name,ITP_ID
 Olivia,Ramacier,[email protected],Los Angeles County Metropolitan Transportation Authority,182
 Eric,Dasmalchi,[email protected],Los Angeles County Metropolitan Transportation Authority,182
-Christian,Suyat,[email protected],Los Angeles County Metropolitan Transportation Authority,182
+Christian,Suyat,[email protected],Los Angeles County Metropolitan Transportation Authority,182

templates/report.html.jinja

@@ -73,7 +73,9 @@
     </div>
 
     <hr>
-
+    {%- macro getJsonForAttribute(variable, attribute) %}
+        {{ variable|map(attribute=attribute)|list|tojson }}
+    {% endmacro -%}
     <div class="flex-1 responsive-prose gap-12 lg:gap-16">
         <h2>Current GTFS Feed Info</h2>
         <div class="flex flex-row lg:flex-row flex-wrap gap-12 !my-12">
@@ -106,7 +108,89 @@
             </div>
         {% endif %}
     </div>
+    {% if has_rt_feed %}
+        <div class="flex-1 responsive-prose">
+            <h2>About the Realtime Completeness Charts</h2>
+            <ul>
+            <li>These charts show the percentage of Scheduled Trips with at least one Trip Update or Vehicle Position message assigned to that trip.</li>
+            <li>High percentages with flat lines are best and indicate large amounts of realtime information for customers.</li>
+            <li>Short drops may indicate schedule inaccuracies, missed runs (buses not on schedule), or hardware failures.</li>
+            </ul>
+        </div>
+    <div class="flex flex-col lg:flex-row gap-12 lg:gap-16 xl:gap-24 2xl:gap-36">
+        <div class="flex-1">
+            <div class="responsive-prose">
+                <h2>Trip Updates Completeness by&nbsp;Day</h2>
+            </div>
+            {% set tu_hours = gtfs_rt_completeness|list %}
+            <div class="hours-chart"
+                 data-dates="{{ getJsonForAttribute (tu_hours, 'service_date')|trim }}"
+                 data-hours="{{ getJsonForAttribute(tu_hours, 'percent_of_trips_with_TU_messages')|trim }}"
+                 data-color="#F6BF16"
+                 data-y-axis-label="% of trips with messages"
+                 data-tooltip-value-label="%"
+                 data-chart-type="area"
+                 data-chart-collabel="trip_update_completeness">
+            </div>
+        </div>
+        <div class="flex-1">
+            <div class="responsive-prose">
+                <h2>Vehicle Positions Completeness by&nbsp;Day</h2>
+            </div>
+            {% set vp_hours = gtfs_rt_completeness|list %}
+            <div class="hours-chart"
+                 data-dates="{{ getJsonForAttribute(vp_hours, 'service_date')|trim }}"
+                 data-hours="{{ getJsonForAttribute(vp_hours, 'percent_of_trips_with_VP_messages')|trim }}"
+                 data-color="#5B559C"
+                 data-y-axis-label="% of trips with messages"
+                 data-tooltip-value-label="percent"
+                 data-chart-type="area"
+                 data-chart-collabel="vehicle_position_completeness">
+            </div>
+        </div>
+    </div>
+        <div class="flex-1 responsive-prose">
+            <h2>About the Median Message Age&nbsp;Charts</h2>
+            <ul>
+            <li>These charts show the daily average age of the median message for Trip Updates and Vehicle Position messages over time.</li>
+            <li>The charts provides insights into how long it takes to get messages from the vehicles to riders.  Lower latencies are better.</li>
+            <li>Higher latencies may indicate outdated hardware or technical failures.</li>
+            </ul>
+        </div>
+    <div class="flex flex-col lg:flex-row gap-12 lg:gap-16 xl:gap-24 2xl:gap-36">
+        <div class="flex-1">
+            <div class="responsive-prose">
+                <h2>Median Trip Update Message Age</h2>
+            </div>
+            {% set tu_age = ave_median_tu_age|list %}
+            <div class="hours-chart"
+                 data-dates="{{ getJsonForAttribute(tu_age, 'service_date')|trim }}"
+                 data-hours="{{ getJsonForAttribute(tu_age, 'avg_median_trip_update_message_age')|trim }}"
+                 data-color="#F6BF16"
+                 data-y-axis-label="Avg TU median message age"
+                 data-tooltip-value-label="seconds"
+                 data-chart-type="line"
+                 data-chart-collabel="avg_median_age">
+            </div>
+        </div>
+        <div class="flex-1">
+            <div class="responsive-prose">
+                <h2>Median Vehicle Positions Message Age</h2>
+            </div>
+            {% set vp_age = ave_median_vp_age|list %}
+            <div class="hours-chart"
+                 data-dates="{{ getJsonForAttribute(vp_age, 'service_date')|trim }}"
+                 data-hours="{{ getJsonForAttribute(vp_age, 'avg_median_vehicle_message_age')|trim }}"
+                 data-color="#5B559C"
+                 data-y-axis-label="Avg VP median message age"
+                 data-tooltip-value-label="seconds"
+                 data-chart-type="line"
+                 data-chart-collabel="avg_median_age">
+            </div>
+        </div>
 
+    </div>
+    {% endif %}
     <div class="responsive-prose max-w-none">
         <div class="my-8 flex gap-4">
             <button type="button"
@@ -173,10 +257,6 @@
 
     <hr>
 
-    {%- macro getJsonForAttribute(variable, attribute) %}
-        {{ variable|map(attribute=attribute)|list|tojson }}
-    {% endmacro -%}
-
     <div class="flex flex-col lg:flex-row gap-12 lg:gap-16 xl:gap-24 2xl:gap-36 mb-8">
         <div class="flex-1 responsive-prose">
             <h2>About the {{ feed_info.date_month }} Daily Service Level&nbsp;Charts</h2>
@@ -202,7 +282,8 @@
             <div class="hours-chart"
                  data-dates="{{ getJsonForAttribute(weekday_hours, 'service_date')|trim }}"
                  data-hours="{{ getJsonForAttribute(weekday_hours, 'ttl_service_hours')|trim }}"
-                 data-color="#136C97">
+                 data-color="#136C97"
+                 data-chart-type="area">
             </div>
         </div>
     </div>
@@ -217,7 +298,8 @@
             <div class="hours-chart"
                  data-dates="{{ getJsonForAttribute(sat_hours, 'service_date')|trim }}"
                  data-hours="{{ getJsonForAttribute(sat_hours, 'ttl_service_hours')|trim }}"
-                 data-color="#F6BF16">
+                 data-color="#F6BF16"
+                 data-chart-type="area">
             </div>
         </div>
 
@@ -230,7 +312,8 @@
             <div class="hours-chart"
                  data-dates="{{ getJsonForAttribute(sun_hours, 'service_date')|trim }}"
                  data-hours="{{ getJsonForAttribute(sun_hours, 'ttl_service_hours')|trim }}"
-                 data-color="#5B559C">
+                 data-color="#5B559C"
+                 data-chart-type="area">
             </div>
         </div>
     </div>

tests/test_report_data.py

@@ -15,6 +15,7 @@
 files = [
     "1_feed_info.json",
     "2_daily_service_hours.json",
+    "2_gtfs_rt_completeness.json",
     "3_routes_changed.json",
     "3_stops_changed.json",
     "4_file_check.json",

website/assets/js/report.js

@@ -1,15 +1,18 @@
-import bb, { bar, area } from 'billboard.js';
+import bb, { bar, area, line } from 'billboard.js';
+
+const twoDecimals = new Intl.NumberFormat('default', { maximumFractionDigits: 2});
 
 // HOURS CHARTS
 // *****************************************************************************
 
 const hoursCharts = document.querySelectorAll('.hours-chart');
-const twoDecimals = new Intl.NumberFormat('default', { maximumFractionDigits: 2});
 
 hoursCharts.forEach((chartEl) => {
     const dates = JSON.parse(chartEl.dataset.dates);
     const hours = JSON.parse(chartEl.dataset.hours);
+    const chartType = chartEl.dataset.chartType || 'line'; // Get chart type from dataset, default to 'line'
     const color = chartEl.dataset.color;
+    const colLabel = chartEl.dataset.chartCollabel || 'hours';
 
     const chart = bb.generate({
         bindto: chartEl,
@@ -21,10 +24,10 @@ hoursCharts.forEach((chartEl) => {
             xFormat: "%Q",
             columns: [
                 ["Date"].concat(dates),
-                ["Hours"].concat(hours),
+                [colLabel].concat(hours),
             ],
             types: {
-                Hours: area(),
+                [colLabel]: chartType === 'area' ? area() : line()
             }
         },
         area: {
@@ -48,7 +51,7 @@ hoursCharts.forEach((chartEl) => {
             y: {
                 label: {
                     position: "outer-middle",
-                    text: "Total service hours"
+                    text: chartEl.dataset.yAxisLabel || "Total service hours" // Default to "Total service hours" if not provided
                 },
                 tick: {
                     culling: {
@@ -70,7 +73,8 @@ hoursCharts.forEach((chartEl) => {
                     year: 'numeric',
                     timeZone: 'UTC',
                 }).format(x),
-                value: (x) => twoDecimals.format(x),
+                // Get the tooltip value label from the chart element's dataset
+                value: (x) => `${twoDecimals.format(x)} ${chartEl.dataset.tooltipValueLabel || 'hours'}`
             },
         },
 
@@ -90,7 +94,6 @@ hoursCharts.forEach((chartEl) => {
     })
 });
 
-
 // CHANGES CHART
 // *****************************************************************************