GTFSRTEntitiesDispatcherJob : suppression de métadonnées anciennes (#…

…2726)

* GTFSRTEntitiesDispatcherJob : suppression de métadonnées anciennes

* Use :day

* Rename file

* Dispatch jobs before deleting

apps/transport/lib/jobs/gtfs_rt_metadata.ex

@@ -1,20 +1,36 @@
 defmodule Transport.Jobs.GTFSRTMetadataDispatcherJob do
   @moduledoc """
-  Job in charge of dispatching multiple `GTFSRTMetadataJob`.
+  Job in charge of:
+  - removing old `DB.ResourceMetadata` for GTFS-RT resources
+  - dispatching multiple `GTFSRTMetadataJob`.
   """
   use Oban.Worker, max_attempts: 3
   import Ecto.Query
-  alias DB.{Dataset, Repo, Resource}
+  alias DB.{Dataset, Repo, Resource, ResourceMetadata}
+
+  @metadata_max_nb_days 90
 
   @impl Oban.Worker
   def perform(%Oban.Job{}) do
     relevant_resources()
     |> Enum.map(&(%{resource_id: &1.id} |> Transport.Jobs.GTFSRTMetadataJob.new()))
     |> Oban.insert_all()
 
+    remove_old_metadata()
+
     :ok
   end
 
+  def remove_old_metadata do
+    recent_limit = DateTime.utc_now() |> DateTime.add(-@metadata_max_nb_days, :day)
+
+    ResourceMetadata
+    |> join(:inner, [rm], r in Resource, on: rm.resource_id == r.id and r.format == "gtfs-rt")
+    |> where([rm, _r], rm.inserted_at < ^recent_limit)
+    |> select([rm, _r], rm)
+    |> Repo.delete_all()
+  end
+
   def relevant_resources do
     Resource
     |> join(:inner, [r], d in Dataset, on: r.dataset_id == d.id)

apps/transport/test/transport/jobs/gtfs_rt_entities_test.exs → apps/transport/test/transport/jobs/gtfs_rt_metadata_test.exs

@@ -29,6 +29,20 @@ defmodule Transport.Test.Transport.Jobs.GTFSRTMetadataJobTest do
       assert :ok == perform_job(GTFSRTMetadataDispatcherJob, %{})
       assert [%Oban.Job{args: %{"resource_id" => ^resource_id}}] = all_enqueued(worker: GTFSRTMetadataJob)
     end
+
+    test "removes old metadata" do
+      resource = insert(:resource, format: "gtfs-rt")
+      gtfs_resource = insert(:resource, format: "gtfs")
+      rm1 = insert(:resource_metadata, resource_id: resource.id, inserted_at: days_ago(30))
+      rm2 = insert(:resource_metadata, resource_id: resource.id, inserted_at: days_ago(89))
+      rm3 = insert(:resource_metadata, resource_id: resource.id, inserted_at: days_ago(91))
+      rm4 = insert(:resource_metadata, resource_id: gtfs_resource.id, inserted_at: days_ago(91))
+
+      assert :ok == perform_job(GTFSRTMetadataDispatcherJob, %{})
+
+      assert [rm1, rm2, nil, rm4] == DB.Repo.reload([rm1, rm2, rm3, rm4])
+      assert resource == DB.Repo.reload(resource)
+    end
   end
 
   describe "GTFSRTMetadataJob" do
@@ -66,4 +80,8 @@ defmodule Transport.Test.Transport.Jobs.GTFSRTMetadataJobTest do
   defp setup_gtfs_rt_feed(url) do
     setup_http_response(url, {:ok, %HTTPoison.Response{status_code: 200, body: File.read!(@sample_file)}})
   end
+
+  defp days_ago(nb) when nb > 0 do
+    DateTime.utc_now() |> DateTime.add(-nb * 24 * 60 * 60)
+  end
 end