DRAFT Update to GTFS schedule validator v7.1 & add tests (#3763, #4378)

.github/workflows/build-gtfs-schedule-validator-image.yml

@@ -56,5 +56,5 @@ jobs:
       - uses: docker/build-push-action@v2
         with:
           context: jobs/gtfs-schedule-validator
-          push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
-          tags: ghcr.io/${{github.repository}}/gtfs-schedule-validator:latest
+          push: true
+          tags: ghcr.io/${{github.repository}}/gtfs-schedule-validator:${{ github.ref == 'refs/heads/main' && 'latest' || 'development'  }}

airflow/.development.env

@@ -50,6 +50,8 @@ COMPOSER_CONTAINER_RUN_AS_HOST_USER=True
 DBT_TARGET=staging
 GOOGLE_CLOUD_PROJECT=cal-itp-data-infra-staging
 POD_LOCATION=us-west2
+POD_CLUSTER_NAME=airflow-jobs-staging
+POD_SECRETS_NAMESPACE=airflow-jobs
 SENTRY_DSN=https://[email protected]/2
 SENTRY_ENVIRONMENT=cal-itp-data-infra-staging
 GTFS_RT_VALIDATOR_JAR=gtfs-realtime-validator-lib-1.0.0-20220223.003525-2.jar

jobs/gtfs-schedule-validator/Dockerfile

@@ -32,10 +32,15 @@ COPY ./gtfs-validator-4.2.0-cli.jar ${V4_2_VALIDATOR_JAR}
 ENV V5_VALIDATOR_JAR=/gtfs-validator-5.0.0-cli.jar
 COPY ./gtfs-validator-5.0.0-cli.jar ${V5_VALIDATOR_JAR}
 
+# v7.1 from https://github.com/MobilityData/gtfs-validator/releases/download/v7.1.0/gtfs-validator-7.1.0-cli.jar
+ENV V7_1_VALIDATOR_JAR=/gtfs-validator-7.1.0-cli.jar
+COPY ./gtfs-validator-7.1.0-cli.jar ${V7_1_VALIDATOR_JAR}
+
 WORKDIR /app
 
 COPY ./pyproject.toml /app/pyproject.toml
 COPY ./poetry.lock /app/poetry.lock
+RUN poetry self add poetry-plugin-export
 RUN poetry export -f requirements.txt --without-hashes --output requirements.txt \
     && pip install -r requirements.txt
 RUN pip install memray

jobs/gtfs-schedule-validator/gtfs_schedule_validator_hourly.py

@@ -42,6 +42,7 @@
 V4_1_VALIDATOR_JAR = os.getenv("V4_1_VALIDATOR_JAR")
 V4_2_VALIDATOR_JAR = os.getenv("V4_2_VALIDATOR_JAR")
 V5_VALIDATOR_JAR = os.getenv("V5_VALIDATOR_JAR")
+V7_1_VALIDATOR_JAR = os.getenv("V7_1_VALIDATOR_JAR")
 
 JAR_DEFAULT = typer.Option(
     default=os.environ.get(SCHEDULE_VALIDATOR_JAR_LOCATION_ENV_KEY),
@@ -165,9 +166,12 @@ def execute_schedule_validator(
     elif extract_ts.date() < pendulum.Date(2024, 3, 27):
         versioned_jar_path = V4_2_VALIDATOR_JAR
         validator_version = "v4.2.0"
-    else:
+    elif extract_ts.date() < pendulum.Date(2025, 9, 1):
         versioned_jar_path = V5_VALIDATOR_JAR
         validator_version = "v5.0.0"
+    else:
+        versioned_jar_path = V7_1_VALIDATOR_JAR
+        validator_version = "v7.1.0"
 
     assert versioned_jar_path
 

jobs/gtfs-schedule-validator/pyproject.toml

@@ -3,6 +3,7 @@ name = "gtfs-schedule-validator"
 version = "0.1.0"
 description = ""
 authors = ["Andrew Vaccaro <[email protected]>"]
+package-mode = false
 
 [tool.poetry.dependencies]
 python = ">=3.8,<3.10"
@@ -18,7 +19,10 @@ python-slugify = "^7.0.0"
 sentry-sdk = "^1.12.1"
 calitp-data-infra = "2024.2.12.post1"
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"
+
+[tool.poetry.group.dev.dependencies]
 pytest = "^5.2"
 mypy = "^0.991"
 types-tqdm = "^4.64.7"

jobs/gtfs-schedule-validator/test_gtfs_schedule_validator.py

@@ -1,7 +1,10 @@
 from pathlib import Path
 
+# import pytest
 import toml
-from gtfs_schedule_validator_hourly import __version__
+from gtfs_schedule_validator_hourly import __version__  # , app
+
+# from typer.testing import CliRunner
 
 
 def test_version():
@@ -20,3 +23,21 @@ def test_versions_are_in_sync():
     package_init_version = __version__
 
     assert package_init_version == pyproject_version
+
+
+# class TestGtfsScheduleValidator:
+#     @pytest.fixture
+#     def runner(self) -> CliRunner:
+#         return CliRunner()
+
+# def test_no_extracts(self, runner):
+#     hour_start = "2020-02-20T20"
+#     result = runner.invoke(
+#         app,
+#         [
+#             "validate-hour",
+#             hour_start,
+#         ],
+#     )
+#     # assert result.exit_code == 0
+#     assert "found 0 extracts to process, exiting" in result.stdout

warehouse/models/intermediate/gtfs_quality/int_gtfs_quality__schedule_validator_rule_details_unioned.sql

@@ -7,6 +7,7 @@ WITH unioned AS (
             ref('gtfs_schedule_validator_rule_details_v4_1_0'),
             ref('gtfs_schedule_validator_rule_details_v4_2_0'),
             ref('gtfs_schedule_validator_rule_details_v5_0_0'),
+            ref('gtfs_schedule_validator_rule_details_v7_1_0'),
         ],
     ) }}
 ),

warehouse/models/mart/gtfs_quality/_mart_gtfs_quality.yml

@@ -96,7 +96,11 @@ models:
               where: validation_validator_version = 'v4.2.0'
           - dbt_utils.accepted_range:
               min_value: "DATE'2024-03-27'"
+              max_value: "DATE'2025-09-01'"
               where: validation_validator_version = 'v5.0.0'
+          - dbt_utils.accepted_range:
+              min_value: "DATE'2025-09-02'"
+              where: validation_validator_version = 'v7.1.0'
       - &schedule_feed_key
         name: feed_key
         tests: