Merge pull request #1750 from cal-itp/1726-mtc-copy-files-script

1726 mtc copy files script

Author: edasmalchi

rt_segment_speeds/52_copy_speedmaps.ipynb

@@ -0,0 +1,356 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "701f2fa1-cdec-4027-a35c-22160c82869d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import geopandas as gpd\n",
+    "import pandas as pd\n",
+    "from calitp_data_analysis.gcs_geopandas import GCSGeoPandas\n",
+    "from shared_utils import catalog_utils, rt_dates, portfolio_utils\n",
+    "from segment_speed_utils import helpers\n",
+    "g = GCSGeoPandas()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "45583f17-cf3c-4580-9231-9e3b410d0c75",
+   "metadata": {},
+   "source": [
+    "# Copy speedmaps data into public bucket to support MTC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "58fe3aeb-5490-4744-a5b5-b315166c9c92",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['2025-09-23',\n",
+       " '2025-09-24',\n",
+       " '2025-09-25',\n",
+       " '2025-10-14',\n",
+       " '2025-10-15',\n",
+       " '2025-10-16']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "relevant_dates = [\n",
+    "    rt_dates.DATES[i]\n",
+    "    for i in [\"sep2025a\", \"sep2025\", \"sep2025b\", \"oct2025a\", \"oct2025\", \"oct2025b\"]\n",
+    "]\n",
+    "relevant_dates"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "698f3c19-afc0-4d32-bdfb-e934a1aebd4d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "gtfs_yml = catalog_utils.get_catalog(\"gtfs_analytics_data\")\n",
+    "aggregated_parent = (\n",
+    "    f\"{gtfs_yml.speedmap_segments.dir}{gtfs_yml.speedmap_segments.segment_timeofday}\"\n",
+    ")\n",
+    "trip_speeds_parent = (\n",
+    "    f\"{gtfs_yml.speedmap_segments.dir}{gtfs_yml.speedmap_segments.stage4}\"\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "5b2dfe84-3231-41e0-8a4c-38d89c339f98",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_shape_timeofday_speedmap_segments_2025-09-23.parquet',\n",
+       " 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_shape_timeofday_speedmap_segments_2025-09-24.parquet',\n",
+       " 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_shape_timeofday_speedmap_segments_2025-09-25.parquet',\n",
+       " 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_shape_timeofday_speedmap_segments_2025-10-14.parquet',\n",
+       " 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_shape_timeofday_speedmap_segments_2025-10-15.parquet',\n",
+       " 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/rollup_singleday/speeds_shape_timeofday_speedmap_segments_2025-10-16.parquet']"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Get aggregated speedmaps segments paths\n",
+    "aggregated_uris = [f\"{aggregated_parent}_{date}.parquet\" for date in relevant_dates]\n",
+    "aggregated_uris"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "66efcd91-9d18-455a-82c7-0124b17ad675",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'date': '2025-09-23',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-09-23.parquet'},\n",
+       " {'date': '2025-09-24',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-09-24.parquet'},\n",
+       " {'date': '2025-09-25',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-09-25.parquet'},\n",
+       " {'date': '2025-10-14',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-10-14.parquet'},\n",
+       " {'date': '2025-10-15',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-10-15.parquet'},\n",
+       " {'date': '2025-10-16',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-10-16.parquet'}]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Get trip level speeds paths\n",
+    "# trip_speeds_uris = [f\"{trip_speeds_parent}_{date}.parquet\" for date in relevant_dates]\n",
+    "trip_speeds_uris = [{\"date\": date, \"uri\": f\"{trip_speeds_parent}_{date}.parquet\"} for date in relevant_dates]\n",
+    "trip_speeds_uris"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d5895977-be56-433d-bc88-5e9385c09916",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true,
+    "tags": []
+   },
+   "source": [
+    "### validation\n",
+    "\n",
+    "Doesn't exactly match script aggregation but we're comfortable these are the right files. Could be revised.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b0ea229d-9d2e-4279-a621-3fe1338b0a3d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "\n",
+    "# # Validate that aggregated speeds match trip speeds in a majority of cases\n",
+    "# GROUP_COLUMNS = [\"time_of_day\", \"segment_id\", \"shape_id\"]\n",
+    "# quantile_to_speed_column = {0.2: \"p20_mph\", 0.5: \"p50_mph\", 0.8: \"p80_mph\"}\n",
+    "# for aggregated_uri, trip_speeds_uri in zip(aggregated_uris, trip_speeds_uris):\n",
+    "#     # read aggregated and precursor data\n",
+    "#     aggregated = g.read_parquet(aggregated_uri)\n",
+    "#     trip_speeds = pd.read_parquet(trip_speeds_uri)\n",
+    "#     # reindex aggregated_speeds and group trip_speeds_grouped so they theoretically have the same data\n",
+    "#     aggregated_speeds = aggregated.set_index(GROUP_COLUMNS)[\n",
+    "#         list(quantile_to_speed_column.values())\n",
+    "#     ]\n",
+    "#     trip_speeds_grouped = trip_speeds.groupby(GROUP_COLUMNS)\n",
+    "#     trip_speeds_percentiles = pd.concat(\n",
+    "#         [\n",
+    "#             trip_speeds_grouped[\"speed_mph\"].quantile(q).rename(name).round(2)\n",
+    "#             for q, name in quantile_to_speed_column.items()\n",
+    "#         ],\n",
+    "#         axis=1,\n",
+    "#     )\n",
+    "#     # the aggregated data appears to contain a subset of the trip speeds data, so only look at that subset\n",
+    "#     trip_speeds_percentiles_subset = trip_speeds_percentiles.loc[\n",
+    "#         aggregated_speeds.index\n",
+    "#     ]\n",
+    "#     # find the rows where at least one speed doesn't match\n",
+    "#     non_equal = (trip_speeds_percentiles_subset != aggregated_speeds).any(axis=1)\n",
+    "    \n",
+    "#     print(\n",
+    "#         f\"for {aggregated_uri}, {non_equal.sum()} out of {len(aggregated_speeds.index)} speeds did not match\"\n",
+    "#     )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dc750c2a-d8b4-4c22-9b10-5d4a3619d8f3",
+   "metadata": {},
+   "source": [
+    "## export"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "9abf5609-17c4-42e0-a24d-3b10b1fbee64",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Map source uris to output uris\n",
+    "output_aggregated_uris = {\n",
+    "    old_uri: f\"gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/{old_uri.split('/')[-1]}\"\n",
+    "    for old_uri in aggregated_uris\n",
+    "}\n",
+    "# output_trip_speeds_uris = {\n",
+    "#     old_uri['uri']: f\"gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/{old_uri.split('/')[-1]}\"\n",
+    "#     for old_uri in trip_speeds_uris\n",
+    "# }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "0bbbbfcc-5e9b-4c11-8ecd-03dcea935fdb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for entry in trip_speeds_uris: entry['output_uri'] = f\"gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/{entry['uri'].split('/')[-1]}\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "5f5d3f1b-ab6f-437b-9877-dda155d50330",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'date': '2025-09-23',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-09-23.parquet',\n",
+       "  'output_uri': 'gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-09-23.parquet'},\n",
+       " {'date': '2025-09-24',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-09-24.parquet',\n",
+       "  'output_uri': 'gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-09-24.parquet'},\n",
+       " {'date': '2025-09-25',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-09-25.parquet',\n",
+       "  'output_uri': 'gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-09-25.parquet'},\n",
+       " {'date': '2025-10-14',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-10-14.parquet',\n",
+       "  'output_uri': 'gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-10-14.parquet'},\n",
+       " {'date': '2025-10-15',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-10-15.parquet',\n",
+       "  'output_uri': 'gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-10-15.parquet'},\n",
+       " {'date': '2025-10-16',\n",
+       "  'uri': 'gs://calitp-analytics-data/data-analyses/rt_segment_speeds/speedmap/speeds_2025-10-16.parquet',\n",
+       "  'output_uri': 'gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-10-16.parquet'}]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trip_speeds_uris"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "423a4975-227d-4426-a6bf-5d57540183fd",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "def add_speeds_identifiers(speeds_df, analysis_date):\n",
+    "    trips = helpers.import_scheduled_trips(analysis_date, columns=['trip_instance_key', 'trip_id', 'gtfs_dataset_key'])\n",
+    "    speeds_df = (speeds_df.merge(trips, on=['trip_instance_key', 'schedule_gtfs_dataset_key'])\n",
+    "                 .pipe(portfolio_utils.standardize_operator_info_for_exports, date=analysis_date)\n",
+    "                )\n",
+    "    return speeds_df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "eca3638b-eac0-47ff-a457-d6564cad7943",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-09-23.parquet\n",
+      "gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-09-24.parquet\n",
+      "gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-09-25.parquet\n",
+      "gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-10-14.parquet\n",
+      "gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-10-15.parquet\n",
+      "gs://calitp-publish-data-analysis/mtc_collab_2025/speeds/speeds_2025-10-16.parquet\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Copy files from the private bucket to the public / mtc-visible bucket\n",
+    "fs = g.gcs_filesystem\n",
+    "# for old_uri, new_uri in output_aggregated_uris.items():\n",
+    "#     fs.copy(old_uri, new_uri)\n",
+    "for uri_group in trip_speeds_uris:\n",
+    "    speeds_df = pd.read_parquet(uri_group['uri'])\n",
+    "    speeds_df = add_speeds_identifiers(speeds_df, analysis_date=uri_group['date'])\n",
+    "    speeds_df.to_parquet(uri_group['output_uri'])\n",
+    "    print(uri_group['output_uri'])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.10"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}