[cli] feat: export GTFS data to database & for ingesting into directions/routing API

apps/admin-cli/.gitignore

@@ -6,3 +6,7 @@
 
 # firebase
 service-account-file.json
+
+# gtfs files
+config/gtfs/*.zip
+config/gtfs/*.txt

apps/admin-cli/src/commands/bus-line/index.ts

@@ -1,5 +1,4 @@
-import { Argument } from 'commander';
-import { program } from '../../structures/command';
+import { Argument, program } from 'commander';
 import updateBusLine from './update';
 import updateAggregateBusLine from './updateAll';
 

apps/admin-cli/src/commands/bus-line/update.ts

@@ -9,6 +9,7 @@ import { getForwardDirectionRelationId } from '../../utils/bus-line';
 import ora from '../../utils/ora';
 import updateAggregateBusLine from './updateAll';
 
+// TODO use data from OSM to populate database in GTFS format
 const updateBusLine = async ({
   id,
   direction,
@@ -28,10 +29,10 @@ const updateBusLine = async ({
     !options.ways && !options.busStops
       ? 'both'
       : options.ways && !options.busStops
-      ? 'ways'
-      : !options.ways && options.busStops
-      ? 'busStops'
-      : 'both';
+        ? 'ways'
+        : !options.ways && options.busStops
+          ? 'busStops'
+          : 'both';
 
   const busLineData = await getBusLine(id);
 
@@ -49,7 +50,7 @@ const updateBusLine = async ({
   const updatedData = { id };
   // use this to force the individual bus-stop/way entries to be replaced,
   // but keep the "top" level keys
-  const mergeFields: string[] = ['id'];
+  const mergeFields: string[] = ['id', 'operator'];
 
   try {
     const forwardDirectionRelationId = getForwardDirectionRelationId(busLineData);
@@ -60,6 +61,16 @@ const updateBusLine = async ({
             (v) => v.type === ElementType.Relation && v.id !== forwardDirectionRelationId
           ) as RelationElement);
 
+    // relations of type route_master cannot be queried with the overpass api (only the children can)
+    // thus, the children relations are the ones which need to have the operator tag
+    if (relationData.tags.operator) {
+      spinner
+        .info(`Bus line ${id} is operated by ${relationData.tags.operator}`)
+        .start('Working...');
+
+      set(updatedData, 'operator', relationData.tags.operator?.toLowerCase());
+    }
+
     if (toUpdate === 'busStops' || toUpdate === 'both') {
       const updatedBusLineStops = relationData.members
         .filter((element) => element.type === ElementType.Node)

apps/admin-cli/src/commands/bus-line/updateAll.ts

@@ -4,6 +4,7 @@ import { getAllBusLineIds } from '../../api/firestore';
 import { firebaseStore } from '../../firebase/config';
 import ora from '../../utils/ora';
 
+// TODO use data from OSM to populate database in GTFS format
 const updateAggregateBusLine = async ({ force }: { force: boolean }): Promise<void> => {
   const spinner = ora('Initialising').start();
 
@@ -51,6 +52,7 @@ const updateAggregateBusLine = async ({ force }: { force: boolean }): Promise<vo
           id: missingBusLineId,
           destination: busLineData.direction['forward'].destination.name,
           origin: busLineData.direction['forward'].origin.name,
+          operator: busLineData.operator,
         };
 
         spinner.text = `${missing.indexOf(missingBusLineId)}/${
@@ -75,10 +77,10 @@ const updateAggregateBusLine = async ({ force }: { force: boolean }): Promise<vo
 
       await batch.commit();
 
-      spinner.succeed(`Done!`).start('Working');
+      spinner.succeed(`Changes have been commited to the database!`).start('Working');
     }
 
-    spinner.succeed(`${missing.length} bus lines updated in the aggregate document`);
+    spinner.succeed(`${missing.length} bus line(s) updated in the aggregate document`);
   } catch (error) {
     spinner.fail(`Failed to update bus lines aggregate document: ${error}`);
   }

apps/admin-cli/src/commands/bus-stop/index.ts

@@ -1,4 +1,4 @@
-import { program } from '../../structures/command';
+import { program } from 'commander';
 import logger from '../../utils/logger';
 import updateAggregateBusStop from './updateAll';
 

apps/admin-cli/src/commands/bus-stop/updateAll.ts

@@ -4,6 +4,7 @@ import { getAllBusStopIds } from '../../api/firestore';
 import { firebaseStore } from '../../firebase/config';
 import ora from '../../utils/ora';
 
+// TODO use data from OSM to populate database in GTFS format
 const updateAggregateBusStop = async ({ force }: { force: boolean }): Promise<void> => {
   const spinner = ora('Initialising').start();
 

apps/admin-cli/src/commands/gtfs/generate.ts

@@ -0,0 +1,26 @@
+import AgencyFile from '../../lib/gtfs/agency';
+import RoutesFile from '../../lib/gtfs/routes';
+import StopsFile from '../../lib/gtfs/stops';
+import ora from '../../utils/ora';
+
+export const generateGtfsFiles = async (_flags: {
+  agency?: boolean;
+  routes?: boolean;
+  stops?: boolean;
+  all: boolean;
+}) => {
+  const spinner = ora('Initialising').start();
+
+  try {
+    spinner.text = 'Working';
+    await Promise.all([
+      AgencyFile.writeToFile(),
+      StopsFile.writeToFile(),
+      RoutesFile.writeToFile(),
+    ]);
+  } catch (error) {
+    spinner.fail(`Failed to generate GTFS files: ${error}`);
+  } finally {
+    spinner.succeed('All done!');
+  }
+};

apps/admin-cli/src/commands/gtfs/index.ts

@@ -0,0 +1,28 @@
+import { program } from 'commander';
+import { generateGtfsFiles } from './generate';
+import { saveGtfsFiles } from './save';
+
+const gtfsCommand = program.command('gtfs').description('commands related to GTFS feeds');
+
+gtfsCommand
+  .command('generate')
+  .description('generate GTFS feed for Mauritius')
+  .option('--agency')
+  .option('--stops')
+  .option('--routes')
+  .option('--all', undefined, false)
+  .action(async function (flags) {
+    // TODO: use flags
+    await generateGtfsFiles(flags);
+  });
+
+gtfsCommand
+  .command('save')
+  .description('Saves GTFS data to the database')
+  .option('--all', undefined, false)
+  .action(async function (flags) {
+    // TODO: use flags
+    await saveGtfsFiles(flags);
+  });
+
+export default gtfsCommand;

apps/admin-cli/src/commands/gtfs/save.ts

@@ -0,0 +1,21 @@
+import AgencyFile from '../../lib/gtfs/agency';
+import RoutesFile from '../../lib/gtfs/routes';
+import StopsFile from '../../lib/gtfs/stops';
+import ora from '../../utils/ora';
+
+export async function saveGtfsFiles(_flags: { all: boolean }) {
+  const spinner = ora('Initialising').start();
+
+  try {
+    spinner.text = 'Working';
+    await Promise.all([
+      AgencyFile.writeToFirestore(),
+      StopsFile.writeToFirestore(),
+      RoutesFile.writeToFirestore(),
+    ]);
+  } catch (error) {
+    spinner.fail(`Failed to save GTFS files: ${error}`);
+  } finally {
+    spinner.succeed('All done!');
+  }
+}

apps/admin-cli/src/index.ts

@@ -3,5 +3,5 @@ import logger from './utils/logger';
 
 loadCommands().then(() => {
   logger.info(`CLI arguments: ${process.argv.slice(2).join(', ')}`);
-  program.parse(process.argv);
+  program.parseAsync(process.argv);
 });

apps/admin-cli/src/interfaces/gtfs/agency.ts

@@ -0,0 +1,43 @@
+export interface Agency extends Record<string, unknown> {
+  /**
+   * Identifies a transit brand which is often synonymous with a transit agency.
+   *
+   * Note that in some cases, such as when a single agency operates multiple separate services, agencies and brands are distinct.
+   * This document uses the term "agency" in place of "brand".
+   * A dataset may contain data from multiple agencies.
+   * This field is required when the dataset contains data for multiple transit agencies, otherwise it is optional.
+   */
+  agency_id?: string;
+  /** Full name of the transit agency. */
+  agency_name: string;
+  /** URL of the transit agency. */
+  agency_url: string;
+  /**
+   * Timezone where the transit agency is located.
+   *
+   * If multiple agencies are specified in the dataset, each must have the same agency_timezone.
+   */
+  agency_timezone: string;
+  /**
+   * Primary language used by this transit agency.
+   *
+   * This field helps GTFS consumers choose capitalization rules and other language-specific settings for the dataset.
+   */
+  agency_lang?: string;
+  /**
+   * A voice telephone number for the specified agency.
+   *
+   * This field is a string value that presents the telephone number as typical for the agency's service area.
+   * It can and should contain punctuation marks to group the digits of the number.
+   * Dialable text (for example, TriMet's 503-238-RIDE) is permitted, but the field must not contain any other descriptive text.
+   */
+  agency_phone?: string;
+  /** URL of a web page that allows a rider to purchase tickets or other fare instruments for that agency online. */
+  agency_fare_url?: string;
+  /**
+   * Email address actively monitored by the agency’s customer service department.
+   *
+   * This email address should be a direct contact point where transit riders can reach a customer service representative at the agency.
+   */
+  agency_email?: string;
+}

apps/admin-cli/src/interfaces/gtfs/index.ts

@@ -0,0 +1,131 @@
+export type GFTSFeedFileName =
+  | 'agency'
+  | 'stops'
+  | 'routes'
+  | 'trips'
+  | 'stop_times'
+  | 'calendar'
+  | 'calendar_dates'
+  | 'fare_attributes'
+  | 'fare_rule'
+  | 'shape'
+  | 'frequencie'
+  | 'transfer'
+  | 'pathway'
+  | 'level'
+  | 'feed_inf'
+  | 'translation'
+  | 'attribution';
+
+export const enum VehicleType {
+  // Standard route types
+  LIGHT_RAIL,
+  METRO,
+  TRAIN,
+  BUS,
+  FERRY,
+  CABLE_CAR,
+  GONDOLA,
+  FUNICULAR,
+
+  TROLLEY_BUS = 11,
+  MONORAIL = 12,
+
+  // Extended route types
+  // https://developers.google.com/transit/gtfs/reference/extended-route-types
+
+  // RAIL
+  GENERIC_RAIL = 100,
+  HIGH_SPEED_RAIL,
+  LONG_DISTANCE_RAIL,
+  INTER_REGIO_RAIL,
+  CAR_TRANSPORT_RAIL,
+  SLEEPER_RAIL,
+  REGIONAL_RAIL,
+  TOURIST_RAIL,
+  RAIL_SHUTTLE,
+  SUBURBAN_RAIL,
+  REPLACEMENT_RAIL,
+  SPECIAL_RAIL,
+  TRUCK_TRANSPORT_RAIL,
+  ALL_RAIL,
+  CROSS_COUNTRY_RAIL,
+  VEHICLE_TRANSPORT_RAIL,
+  RACK_AND_PINION_RAIL,
+  ADDITIONAL_RAIL,
+
+  // COACH
+  GENERIC_COACH = 200,
+  INTERNATIONAL_COACH,
+  NATIONAL_COACH,
+  SHUTTLE_COACH,
+  REGIONAL_COACH,
+  SPECIAL_COACH,
+  SIGHTSEEING_COACH,
+  TOURIST_COACH,
+  COMMUTER_COACH,
+  ALL_COACH,
+
+  // URBAN_RAIL
+  GENERIC_URBAN_RAIL = 400,
+  METRO_RAIL,
+  UNDERGROUND,
+  URBAN_RAIL,
+  ALL_URBAN_RAIL,
+  URBAN_MONORAIL,
+
+  // BUS
+  GENERIC_BUS = 700,
+  REGIONAL_BUS,
+  EXPRESS_BUS,
+  STOPPING_BUS,
+  LOCAL_BUS,
+  NIGHT_BUS,
+  POST_BUS,
+  SPECIAL_NEEDS_BUS,
+  MOBILITY_BUS,
+  DISABILITY_BUS,
+  SIGHT_SEEING_BUS,
+  SHUTTLE_BUS,
+  SCHOOL_BUS,
+  PUBLIC_SERVICE_BUS,
+  RAIL_REPLACEMENT_BUS,
+  DEMAND_RESPONSIVE_BUS,
+  ALL_BUS,
+
+  // TROLLEY BUS
+  GENERIC_TROLLEY_BUS = 800,
+
+  // TRAM
+  TRAM = 900,
+  CITY_TRAM,
+  LOCAL_TRAM,
+  REGIONAL_TRAM,
+  SIGHTSEEING_TRAM,
+  SHUTTLE_TRAM,
+  ALL_TRAM,
+
+  WATER_TRANSPORT = 1000,
+
+  AIR = 1100,
+
+  GENERIC_FERRY = 1200,
+
+  AERIALWAY = 1300,
+
+  GENERIC_FUNICULAR = 1400,
+
+  // TAXI
+  TAXI = 1500,
+  COMMUNAL_TAXI,
+  WATER_TAXI,
+  RAIL_TAXI,
+  BIKE_TAXI,
+  LICENSED_TAXI,
+  PRIVATE_HIRE_TAXI,
+  ALL_TAXI,
+
+  // MISCELLANEOUS
+  MISC = 1700,
+  HORSE_DRAWN_CARRIAGE = 1702,
+}