add latest dataset metadata to feed response (#262)

Add latest dataset metadata to feed response. Rename database column

api/src/feeds/filters/gtfs_dataset_filter.py

@@ -6,8 +6,8 @@
 
 
 class GtfsDatasetFilter(Filter):
-    download_date__lte: Optional[datetime]
-    download_date__gte: Optional[datetime]
+    downloaded_at__lte: Optional[datetime]
+    downloaded_at__gte: Optional[datetime]
 
     class Constants(Filter.Constants):
         model = Gtfsdataset

api/src/feeds/impl/datasets_api_impl.py

@@ -104,8 +104,8 @@ def get_datasets_gtfs(query: Query, limit: int = None, offset: int = None) -> Li
                 feed_id=feed_ids[0],
                 hosted_url=database_gtfs_dataset.hosted_url,
                 note=database_gtfs_dataset.note,
-                downloaded_at=database_gtfs_dataset.download_date.isoformat()
-                if database_gtfs_dataset.download_date
+                downloaded_at=database_gtfs_dataset.downloaded_at.isoformat()
+                if database_gtfs_dataset.downloaded_at
                 else None,
                 hash=database_gtfs_dataset.hash,
                 components=[component for component in components if component is not None],

api/src/feeds/impl/feeds_api_impl.py

@@ -1,6 +1,6 @@
 import json
 from typing import List, Type, Set, Union
-
+from datetime import datetime
 from fastapi import HTTPException
 from sqlalchemy.orm import Query, aliased
 
@@ -215,7 +215,12 @@ def _get_order_by_key(order_by: list[str] = None):
                 (None, None),
             )
             if latest_dataset:
-                api_dataset = LatestDataset(id=latest_dataset.stable_id, hosted_url=latest_dataset.hosted_url)
+                api_dataset = LatestDataset(
+                    id=latest_dataset.stable_id,
+                    downloaded_at=latest_dataset.downloaded_at.isoformat() if latest_dataset.downloaded_at else None,
+                    hash=latest_dataset.hash,
+                    hosted_url=latest_dataset.hosted_url,
+                )
                 if bounding_box:
                     coordinates = json.loads(bounding_box)["coordinates"][0]
                     api_dataset.bounding_box = BoundingBox(
@@ -319,14 +324,14 @@ def get_gtfs_feed_datasets(
         latest: bool,
         limit: int,
         offset: int,
-        downloaded_date_gte: str,
-        downloaded_date_lte: str,
+        downloaded_at_gte: str,
+        downloaded_at_lte: str,
     ) -> List[GtfsDataset]:
         """Get a list of datasets related to a feed."""
         # getting the bounding box as JSON to make it easier to process
         query = GtfsDatasetFilter(
-            download_date__lte=downloaded_date_lte,
-            download_date__gte=downloaded_date_gte,
+            downloaded_at__lte=datetime.fromisoformat(downloaded_at_lte) if downloaded_at_lte else None,
+            downloaded_at__gte=datetime.fromisoformat(downloaded_at_gte) if downloaded_at_gte else None,
         ).filter(DatasetsApiImpl.create_dataset_query().filter(Feed.stable_id == id))
 
         if latest:

docs/DatabaseCatalogAPI.yaml

@@ -180,8 +180,8 @@ paths:
         - $ref: "#/components/parameters/latestQueryParam"
         - $ref: "#/components/parameters/limitQueryParam"
         - $ref: "#/components/parameters/offset"
-        - $ref: "#/components/parameters/downloaded_date_gte"
-        - $ref: "#/components/parameters/downloaded_date_lte"
+        - $ref: "#/components/parameters/downloaded_at_gte"
+        - $ref: "#/components/parameters/downloaded_at_lte"
 
       security:
         -  Authentication: []
@@ -371,6 +371,15 @@ components:
           example: https://storage.googleapis.com/storage/v1/b/mdb-latest/o/us-maine-casco-bay-lines-gtfs-1.zip?alt=media
         bounding_box:
           $ref: "#/components/schemas/BoundingBox"
+        downloaded_at:
+          description: The date and time the dataset was downloaded from the producer, in ISO 8601 format.
+          type: string
+          example: 2023-07-10T22:06:00Z
+          format: datetime
+        hash:
+          description: A MD5 hash of the dataset.
+          type: string
+          example: a_long_sha1_hash
 
 #    Have to put the enum inline because of a bug in openapi-generator
 #    EntityTypes:
@@ -681,15 +690,15 @@ components:
       schema:
         type: string
         example: Los Angeles
-    downloaded_date_gte:
-      name: downloaded_date_gte
+    downloaded_at_gte:
+      name: downloaded_at_gte
       in: query
       description: Filter feed datasets with downloaded date greater or equal to given date.
       schema:
         type: string
         example: 2023-07-00T22:06:00Z
-    downloaded_date_lte:
-      name: downloaded_date_lte
+    downloaded_at_lte:
+      name: downloaded_at_lte
       in: query
       description: Filter feed datasets with downloaded date less or equal to given date.
       schema:

functions-python/batch_datasets/tests/conftest.py

@@ -63,7 +63,7 @@ def populate_database():
             hosted_url=fake.url(),
             note=fake.sentence(),
             hash=fake.sha256(),
-            download_date=datetime.utcnow(),
+            downloaded_at=datetime.utcnow(),
             stable_id=fake.uuid4(),
         )
         session.add(gtfs_dataset)

functions-python/batch_process_dataset/src/main.py

@@ -199,7 +199,7 @@ def create_dataset(self, dataset_file: DatasetFile):
                 bounding_box=None,
                 note=None,
                 hash=dataset_file.file_sha256_hash,
-                download_date=func.now(),
+                downloaded_at=func.now(),
                 hosted_url=dataset_file.hosted_url,
             )
             if latest_dataset:

functions-python/batch_process_dataset/tests/conftest.py

@@ -63,7 +63,7 @@ def populate_database():
             hosted_url=fake.url(),
             note=fake.sentence(),
             hash=fake.sha256(),
-            download_date=datetime.utcnow(),
+            downloaded_at=datetime.utcnow(),
             stable_id=fake.uuid4(),
         )
         session.add(gtfs_dataset)

liquibase/changelog.xml

@@ -17,4 +17,5 @@
     <include file="changes/feat_149.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_240.sql" relativeToChangelogFile="true"/>
     <include file="changes/feat_263.sql" relativeToChangelogFile="true"/>
+     <include file="changes/feat_66.sql" relativeToChangelogFile="true"/>
 </databaseChangeLog>

liquibase/changes/feat_66.sql

@@ -0,0 +1,2 @@
+ALTER TABLE GTFSDataset
+RENAME COLUMN download_date TO downloaded_at;

scripts/api-tests.sh

@@ -91,6 +91,7 @@ cat $ABS_SCRIPTPATH/../config/.env.local > $ABS_SCRIPTPATH/../.env
 execute_tests() {
   printf "\nExecuting tests in $1\n"
   cd $ABS_SCRIPTPATH/$1/ || exit 1
+  cp $ABS_SCRIPTPATH/../.env $ABS_SCRIPTPATH/$1/.env
   pip3 install --disable-pip-version-check virtualenv >/dev/null
   python3 -m virtualenv venv >/dev/null
   venv/bin/python -m pip install --disable-pip-version-check -r requirements.txt >/dev/null