fix(GlobalDataCache): trigger timed fetch on startup (#208)

* fix(GlobalDataCache): trigger timed fetch on startup

* fix: remove unused imports

* remove unneeded conditional check

* fix start_global_cache typo

.github/workflows/ci.yml

@@ -119,6 +119,9 @@ jobs:
         run: docker compose build
       - name: docker compose up
         run: docker compose up --wait
+        env:
+          API_KEY: ${{secrets.API_KEY}}
+          API_URL: ${{secrets.API_URL}}
       - name: ensure running properly
         run: docker compose exec --no-TTY mobile-app-backend wget --spider -S http://localhost:4000/
       - name: show docker container logs

config/test.exs

@@ -11,6 +11,8 @@ config :mobile_app_backend, MobileAppBackendWeb.Endpoint,
 # isolated fake processes
 config :mobile_app_backend, start_stream_stores?: false
 
+config :mobile_app_backend, start_global_cache?: false
+
 # Print only warnings and errors during test
 config :logger, level: :warning
 

lib/mobile_app_backend/application.ex

@@ -9,26 +9,38 @@ defmodule MobileAppBackend.Application do
   def start(_type, _args) do
     Logger.add_handlers(:mobile_app_backend)
 
-    children = [
-      MobileAppBackendWeb.Telemetry,
-      {DNSCluster,
-       query: Application.get_env(:mobile_app_backend, :dns_cluster_query) || :ignore},
-      Supervisor.child_spec({Phoenix.PubSub, name: MobileAppBackend.PubSub}, id: :general_pubsub),
-      {Finch,
-       name: Finch.CustomPool,
-       pools: %{
-         :default => [size: 200, count: 10, start_pool_metrics?: true]
-       }},
-      {MBTAV3API.ResponseCache, []},
-      MBTAV3API.Supervisor,
-      {MobileAppBackend.FinchPoolHealth, pool_name: Finch.CustomPool},
-      MobileAppBackend.MapboxTokenRotator,
-      MobileAppBackend.Predictions.Registry,
-      MobileAppBackend.Predictions.PubSub,
-      MobileAppBackend.GlobalDataCache,
-      # Start to serve requests, typically the last entry
-      MobileAppBackendWeb.Endpoint
-    ]
+    start_global_cache? =
+      Application.get_env(:mobile_app_backend, :start_global_cache?, true)
+
+    children =
+      [
+        MobileAppBackendWeb.Telemetry,
+        {DNSCluster,
+         query: Application.get_env(:mobile_app_backend, :dns_cluster_query) || :ignore},
+        Supervisor.child_spec({Phoenix.PubSub, name: MobileAppBackend.PubSub},
+          id: :general_pubsub
+        ),
+        {Finch,
+         name: Finch.CustomPool,
+         pools: %{
+           :default => [size: 200, count: 10, start_pool_metrics?: true]
+         }},
+        {MBTAV3API.ResponseCache, []},
+        MBTAV3API.Supervisor,
+        {MobileAppBackend.FinchPoolHealth, pool_name: Finch.CustomPool},
+        MobileAppBackend.MapboxTokenRotator,
+        MobileAppBackend.Predictions.Registry,
+        MobileAppBackend.Predictions.PubSub
+      ] ++
+        if start_global_cache? do
+          [MobileAppBackend.GlobalDataCache]
+        else
+          []
+        end ++
+        [
+          # Start to serve requests, typically the last entry
+          MobileAppBackendWeb.Endpoint
+        ]
 
     :ok = MobileAppBackend.FinchTelemetryLogger.attach()
 

lib/mobile_app_backend/global_data_cache.ex

@@ -21,43 +21,100 @@ defmodule MobileAppBackend.GlobalDataCache do
         }
 
   defmodule State do
-    defstruct [:key, :update_ms]
+    defstruct [:key, :update_ms, :first_update_ms]
   end
 
-  def default_key, do: __MODULE__
+  @callback default_key :: key()
+  @callback get_data(key()) :: data()
 
-  def start_link(opts) do
+  def start_link(opts \\ []) do
+    Application.get_env(
+      :mobile_app_backend,
+      MobileAppBackend.GlobalDataCache.Module,
+      MobileAppBackend.GlobalDataCache.Impl
+    ).start_link(opts)
+  end
+
+  def handle_info(msg, state) do
+    Application.get_env(
+      :mobile_app_backend,
+      MobileAppBackend.GlobalDataCache.Module,
+      MobileAppBackend.GlobalDataCache.Impl
+    ).handle_info(msg, state)
+  end
+
+  def init(opts \\ []) do
+    Application.get_env(
+      :mobile_app_backend,
+      MobileAppBackend.GlobalDataCache.Module,
+      MobileAppBackend.GlobalDataCache.Impl
+    ).init(opts)
+  end
+
+  def default_key do
+    Application.get_env(
+      :mobile_app_backend,
+      MobileAppBackend.GlobalDataCache.Module,
+      MobileAppBackend.GlobalDataCache.Impl
+    ).default_key()
+  end
+
+  def get_data(key \\ default_key()) do
+    Application.get_env(
+      :mobile_app_backend,
+      MobileAppBackend.GlobalDataCache.Module,
+      MobileAppBackend.GlobalDataCache.Impl
+    ).get_data(key)
+  end
+end
+
+defmodule MobileAppBackend.GlobalDataCache.Impl do
+  use GenServer
+  alias MBTAV3API.{JsonApi, Repository}
+  alias MobileAppBackend.GlobalDataCache
+  alias MobileAppBackend.GlobalDataCache.State
+
+  @behaviour GlobalDataCache
+
+  @impl true
+  def default_key, do: MobileAppBackend.GlobalDataCache
+
+  @spec start_link(keyword()) :: :ignore | {:error, any()} | {:ok, pid()}
+  def start_link(opts \\ []) do
     opts = Keyword.merge([key: default_key()], opts)
-    GenServer.start_link(__MODULE__, opts)
+    GenServer.start_link(GlobalDataCache, opts)
   end
 
-  @spec get_data(key()) :: data()
+  @impl true
   def get_data(key \\ default_key()) do
     :persistent_term.get(key, nil) || update_data(key)
   end
 
   @impl GenServer
-  def init(opts) do
-    opts = Keyword.merge(Application.get_env(:mobile_app_backend, __MODULE__), opts)
+  def init(opts \\ []) do
+    opts = Keyword.merge(Application.get_env(:mobile_app_backend, GlobalDataCache), opts)
+    first_update_ms = opts[:first_update_ms] || :timer.seconds(1)
 
     state = %State{
       key: opts[:key],
       update_ms: opts[:update_ms] || :timer.minutes(5)
     }
 
+    Process.send_after(self(), :recalculate, first_update_ms)
+
     {:ok, state}
   end
 
   @impl GenServer
-  def handle_info(:recalculate, %State{} = state) do
+  def handle_info(:recalculate, %State{update_ms: update_ms} = state) do
     update_data(state.key)
 
-    Process.send_after(self(), :recalculate, state.update_ms)
+    Process.send_after(self(), :recalculate, update_ms)
 
     {:noreply, state}
   end
 
-  @spec update_data(key()) :: data()
+  @spec update_data(GlobalDataCache.key()) :: GlobalDataCache.data()
   defp update_data(key) do
     stops = fetch_stops()
 

lib/mobile_app_backend_web/controllers/global_controller.ex

@@ -3,10 +3,7 @@ defmodule MobileAppBackendWeb.GlobalController do
   alias MobileAppBackend.GlobalDataCache
 
   def show(conn, _params) do
-    cache_key =
-      Map.get(conn.assigns, :global_cache_key_for_testing, GlobalDataCache.default_key())
-
-    global_data = GlobalDataCache.get_data(cache_key)
+    global_data = GlobalDataCache.get_data()
 
     json(conn, global_data)
   end

test/mobile_app_backend/global_data_cache_test.exs

@@ -1,5 +1,5 @@
 defmodule MobileAppBackend.GlobalDataCacheTest do
-  use HttpStub.Case, async: true
+  use HttpStub.Case
   alias MobileAppBackend.GlobalDataCache
 
   test "gets data" do
@@ -87,4 +87,44 @@ defmodule MobileAppBackend.GlobalDataCacheTest do
              }
            } = lines
   end
+
+  describe "init/1" do
+    test "sends recalculation message" do
+      cache_key = make_ref()
+
+      :persistent_term.put(cache_key, %{
+        lines: %{},
+        pattern_ids_by_stop: %{},
+        routes: %{},
+        route_patterns: %{},
+        stops: %{},
+        trips: %{}
+      })
+
+      GlobalDataCache.init(key: cache_key, first_update_ms: 10)
+      assert_receive :recalculate, 2_000
+    end
+  end
+
+  describe "handle_info/1" do
+    test "sends another recalculate message" do
+      cache_key = make_ref()
+
+      :persistent_term.put(cache_key, %{
+        lines: %{},
+        pattern_ids_by_stop: %{},
+        routes: %{},
+        route_patterns: %{},
+        stops: %{},
+        trips: %{}
+      })
+
+      GlobalDataCache.handle_info(:recalculate, %MobileAppBackend.GlobalDataCache.State{
+        key: cache_key,
+        update_ms: 10
+      })
+
+      assert_receive :recalculate
+    end
+  end
 end

test/mobile_app_backend_web/controllers/global_controller_test.exs

@@ -1,70 +1,82 @@
 defmodule MobileAppBackendWeb.GlobalControllerTest do
   use HttpStub.Case
   use MobileAppBackendWeb.ConnCase
+  import Mox
+  import Test.Support.Helpers
 
   describe "GET /api/global" do
+    setup do
+      verify_on_exit!()
+
+      reassign_env(
+        :mobile_app_backend,
+        MobileAppBackend.GlobalDataCache.Module,
+        GlobalDataCacheMock
+      )
+    end
+
     test "retrieves all stop and route info from the V3 API", %{conn: conn} do
-      cache_key = make_ref()
-
-      :persistent_term.put(cache_key, %{
-        lines: %{
-          "line-Red" => %MBTAV3API.Line{
-            id: "line-Red",
-            color: "DA291C",
-            long_name: "Red Line",
-            short_name: "",
-            sort_order: 10_010,
-            text_color: "FFFFFF"
-          }
-        },
-        pattern_ids_by_stop: %{"70076" => ["Red-1-1", "Red-3-1"]},
-        routes: %{
-          "Red" => %MBTAV3API.Route{
-            id: "Red",
-            color: "DA291C",
-            direction_destinations: ["Ashmont/Braintree", "Alewife"],
-            direction_names: ["South", "North"],
-            line_id: "line-Red",
-            long_name: "Red Line",
-            type: :heavy_rail
-          }
-        },
-        route_patterns: %{
-          "Red-1-1" => %MBTAV3API.RoutePattern{
-            id: "Red-1-1",
-            direction_id: 1,
-            name: "Ashmont - Alewife",
-            representative_trip_id: "canonical-Red-C2-1",
-            route_id: "Red",
-            sort_order: 100_101_001
-          }
-        },
-        stops: %{
-          "place-pktrm" => %MBTAV3API.Stop{
-            id: "place-pktrm",
-            latitude: 42.356395,
-            location_type: :station,
-            longitude: -71.062424,
-            name: "Park Street",
-            child_stop_ids: ["70076"],
-            connecting_stop_ids: ["10000"]
+      GlobalDataCacheMock
+      |> expect(:default_key, fn -> :default_key end)
+      |> expect(:get_data, fn _ ->
+        %{
+          lines: %{
+            "line-Red" => %MBTAV3API.Line{
+              id: "line-Red",
+              color: "DA291C",
+              long_name: "Red Line",
+              short_name: "",
+              sort_order: 10_010,
+              text_color: "FFFFFF"
+            }
           },
-          "70076" => %MBTAV3API.Stop{
-            id: "70076",
-            name: "Park Street",
-            location_type: :stop,
-            parent_station_id: "place-pktrm"
-          }
-        },
-        trips: %{
-          "canonical-Red-C2-1" => %MBTAV3API.Trip{
-            headsign: "Alewife",
-            stop_ids: 70_094..70_062//-2 |> Enum.map(&to_string/1)
+          pattern_ids_by_stop: %{"70076" => ["Red-1-1", "Red-3-1"]},
+          routes: %{
+            "Red" => %MBTAV3API.Route{
+              id: "Red",
+              color: "DA291C",
+              direction_destinations: ["Ashmont/Braintree", "Alewife"],
+              direction_names: ["South", "North"],
+              line_id: "line-Red",
+              long_name: "Red Line",
+              type: :heavy_rail
+            }
+          },
+          route_patterns: %{
+            "Red-1-1" => %MBTAV3API.RoutePattern{
+              id: "Red-1-1",
+              direction_id: 1,
+              name: "Ashmont - Alewife",
+              representative_trip_id: "canonical-Red-C2-1",
+              route_id: "Red",
+              sort_order: 100_101_001
+            }
+          },
+          stops: %{
+            "place-pktrm" => %MBTAV3API.Stop{
+              id: "place-pktrm",
+              latitude: 42.356395,
+              location_type: :station,
+              longitude: -71.062424,
+              name: "Park Street",
+              child_stop_ids: ["70076"],
+              connecting_stop_ids: ["10000"]
+            },
+            "70076" => %MBTAV3API.Stop{
+              id: "70076",
+              name: "Park Street",
+              location_type: :stop,
+              parent_station_id: "place-pktrm"
+            }
+          },
+          trips: %{
+            "canonical-Red-C2-1" => %MBTAV3API.Trip{
+              headsign: "Alewife",
+              stop_ids: 70_094..70_062//-2 |> Enum.map(&to_string/1)
+            }
           }
         }
-      })
-
-      conn = Plug.Conn.assign(conn, :global_cache_key_for_testing, cache_key)
+      end)
 
       conn = get(conn, "/api/global")
       stop_response = json_response(conn, 200)