fix: failing integration tests (#545)

api/src/feeds/impl/feeds_api_impl.py

@@ -1,12 +1,16 @@
-from typing import List, Union
 from datetime import datetime
+from typing import List, Union
+
+from sqlalchemy.orm import joinedload
 
 from database.database import Database
 from database_gen.sqlacodegen_models import (
     Feed,
     Gtfsdataset,
     Gtfsfeed,
     Gtfsrealtimefeed,
+    Location,
+    Validationreport,
 )
 from feeds.filters.feed_filter import FeedFilter
 from feeds.filters.gtfs_dataset_filter import GtfsDatasetFilter
@@ -71,6 +75,7 @@ def get_feeds(
         feed_query = feed_filter.filter(Database().get_query_model(Feed))
         # Results are sorted by provider
         feed_query = feed_query.order_by(Feed.provider, Feed.stable_id)
+        feed_query = feed_query.options(*BasicFeedImpl.get_joinedload_options())
         if limit is not None:
             feed_query = feed_query.limit(limit)
         if offset is not None:
@@ -155,6 +160,13 @@ def get_gtfs_feeds(
             ),
         )
         gtfs_feed_query = gtfs_feed_filter.filter(Database().get_query_model(Gtfsfeed))
+
+        gtfs_feed_query = gtfs_feed_query.outerjoin(Location, Feed.locations).options(
+            joinedload(Gtfsfeed.gtfsdatasets)
+            .joinedload(Gtfsdataset.validation_reports)
+            .joinedload(Validationreport.notices),
+            *BasicFeedImpl.get_joinedload_options(),
+        )
         gtfs_feed_query = gtfs_feed_query.order_by(Gtfsfeed.provider, Gtfsfeed.stable_id)
         gtfs_feed_query = DatasetsApiImpl.apply_bounding_filtering(
             gtfs_feed_query, dataset_latitudes, dataset_longitudes, bounding_filter_method
@@ -211,6 +223,11 @@ def get_gtfs_rt_feeds(
             ),
         )
         gtfs_rt_feed_query = gtfs_rt_feed_filter.filter(Database().get_query_model(Gtfsrealtimefeed))
+        gtfs_rt_feed_query = gtfs_rt_feed_query.outerjoin(Location, Feed.locations).options(
+            *BasicFeedImpl.get_joinedload_options(),
+            joinedload(Gtfsrealtimefeed.entitytypes),
+            joinedload(Gtfsrealtimefeed.gtfs_feeds),
+        )
         gtfs_rt_feed_query = gtfs_rt_feed_query.order_by(Gtfsrealtimefeed.provider, Gtfsrealtimefeed.stable_id)
         if limit is not None:
             gtfs_rt_feed_query = gtfs_rt_feed_query.limit(limit)

api/src/feeds/impl/models/basic_feed_impl.py

@@ -1,3 +1,6 @@
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.strategy_options import _AbstractLoad
+
 from database_gen.sqlacodegen_models import Feed
 from feeds.impl.models.external_id_impl import ExternalIdImpl
 from feeds.impl.models.redirect_impl import RedirectImpl
@@ -41,6 +44,11 @@ def from_orm(cls, feed: Feed | None) -> BasicFeed | None:
             redirects=sorted([RedirectImpl.from_orm(item) for item in feed.redirectingids], key=lambda x: x.target_id),
         )
 
+    @staticmethod
+    def get_joinedload_options() -> [_AbstractLoad]:
+        """Returns common joinedload options for feeds queries."""
+        return [joinedload(Feed.locations), joinedload(Feed.externalids), joinedload(Feed.redirectingids)]
+
 
 class BasicFeedImpl(BaseFeedImpl, BasicFeed):
     """Implementation of the `BasicFeed` model.

api/tests/unittest/test_feeds.py

@@ -77,8 +77,10 @@ def test_feeds_get(client: TestClient, mocker):
     mock_filter = mocker.patch.object(FeedFilter, "filter")
     mock_filter_offset = Mock()
     mock_filter_order_by = Mock()
+    mock_options = Mock()
     mock_filter.return_value.order_by.return_value = mock_filter_order_by
-    mock_filter_order_by.offset.return_value = mock_filter_offset
+    mock_filter_order_by.options.return_value = mock_options
+    mock_options.offset.return_value = mock_filter_offset
     # Target is set to None as deep copy is failing for unknown reasons
     # At the end of the test, the target is set back to the original value
     mock_feed.redirectingids[0].target = None

integration-tests/src/endpoints/datasets.py

@@ -1,6 +1,3 @@
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-import gtfs_kit
 import pandas
 from rich.table import Table
 
@@ -13,38 +10,30 @@ def __init__(self, file_path, access_token, url, progress, console):
 
     def test_all_datasets(self):
         warnings = []
-        with ThreadPoolExecutor(max_workers=3) as executor:
-            # Prepare a list of feed IDs to process
-            feed_ids = self.gtfs_feeds.mdb_source_id.values
-
-            # Create a progress task
-            task = self.progress.add_task(
-                "[yellow]Validating the feeds' latest datasets...[/yellow]",
-                total=len(feed_ids),
-            )
+        # Prepare a list of feed IDs to process
+        feed_ids = self.gtfs_feeds.mdb_source_id.values
+
+        # Create a progress task
+        task = self.progress.add_task(
+            "[yellow]Validating the feeds' latest datasets...[/yellow]",
+            total=len(feed_ids),
+        )
 
-            # Map each future to its feed ID
-            future_to_feed_id = {
-                executor.submit(self.validate_feed, feed_id): feed_id
-                for feed_id in feed_ids
-            }
-
-            for future in as_completed(future_to_feed_id):
-                feed_id = future_to_feed_id[future]
-                warning_entry = future.result()
-                if warning_entry is not None:
-                    warnings.append(warning_entry)
-                    self.console.log(
-                        f"Feed '{warning_entry['stable_id']}' has a related [yellow]warning[/yellow]: "
-                        f"{warning_entry['Warning Details']}"
-                    )
-                else:
-                    self.console.log(
-                        f"Feed 'mdb-{feed_id}' has a valid latest dataset :white_check_mark:"
-                    )
-
-                # Update the progress bar
-                self.progress.update(task, advance=1)
+        for feed_id in feed_ids:
+            warning_entry = self.validate_feed(feed_id)
+            if warning_entry is not None:
+                warnings.append(warning_entry)
+                self.console.log(
+                    f"Feed '{warning_entry['stable_id']}' has a related [yellow]warning[/yellow]: "
+                    f"{warning_entry['Warning Details']}"
+                )
+            else:
+                self.console.log(
+                    f"Feed 'mdb-{feed_id}' has a valid latest dataset :white_check_mark:"
+                )
+
+            # Update the progress bar
+            self.progress.update(task, advance=1)
 
         if warnings:
             # If there were warning, log them as before
@@ -100,14 +89,6 @@ def _validate_dataset(datasets, status_code):
             warning_code = DatasetValidationWarning.NO_DATASET.name
             warning_detail = DatasetValidationWarning.NO_DATASET.value
             raise Exception(f"{warning_code}: {warning_detail}")
-        latest_dataset = datasets[0]
-        try:
-            gtfs_kit.read_feed(latest_dataset["hosted_url"], "km")
-        except Exception as e:
-            raise Exception(
-                f"{DatasetValidationWarning.INVALID_DATASET.name}: {DatasetValidationWarning.INVALID_DATASET.value} -- "
-                f"{e}"
-            )
 
     @staticmethod
     def _create_validation_report_entry(stable_id, warning_details, status_code=None):

integration-tests/src/endpoints/integration_tests.py

@@ -22,7 +22,6 @@ class DatasetValidationWarning(Enum):
     NOT_FOUND = "Feed not returned from the API"
     API_ERROR = "API returned an error code"
     NO_DATASET = "No dataset found for feed"
-    INVALID_DATASET = "Invalid dataset format"
 
 
 class IntegrationTests: