From 3beddf6452742d8bb2c846d77faf0a8f94443c84 Mon Sep 17 00:00:00 2001
From: Antoine Augusti <antoine.augusti@transport.data.gouv.fr>
Date: Mon, 17 Jul 2023 11:04:26 +0200
Subject: [PATCH] =?UTF-8?q?GTFSRTEntitiesDispatcherJob=20:=20suppression?=
 =?UTF-8?q?=20de=20m=C3=A9tadonn=C3=A9es=20anciennes=20(#2726)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* GTFSRTEntitiesDispatcherJob : suppression de métadonnées anciennes

* Use :day

* Rename file

* Dispatch jobs before deleting
---
 apps/transport/lib/jobs/gtfs_rt_metadata.ex   | 20 +++++++++++++++++--
 ...ies_test.exs => gtfs_rt_metadata_test.exs} | 18 +++++++++++++++++
 2 files changed, 36 insertions(+), 2 deletions(-)
 rename apps/transport/test/transport/jobs/{gtfs_rt_entities_test.exs => gtfs_rt_metadata_test.exs} (76%)

diff --git a/apps/transport/lib/jobs/gtfs_rt_metadata.ex b/apps/transport/lib/jobs/gtfs_rt_metadata.ex
index fc98e3180f..fd2904a3b1 100644
--- a/apps/transport/lib/jobs/gtfs_rt_metadata.ex
+++ b/apps/transport/lib/jobs/gtfs_rt_metadata.ex
@@ -1,10 +1,14 @@
 defmodule Transport.Jobs.GTFSRTMetadataDispatcherJob do
   @moduledoc """
-  Job in charge of dispatching multiple `GTFSRTMetadataJob`.
+  Job in charge of:
+  - removing old `DB.ResourceMetadata` for GTFS-RT resources
+  - dispatching multiple `GTFSRTMetadataJob`.
   """
   use Oban.Worker, max_attempts: 3
   import Ecto.Query
-  alias DB.{Dataset, Repo, Resource}
+  alias DB.{Dataset, Repo, Resource, ResourceMetadata}
+
+  @metadata_max_nb_days 90
 
   @impl Oban.Worker
   def perform(%Oban.Job{}) do
@@ -12,9 +16,21 @@ defmodule Transport.Jobs.GTFSRTMetadataDispatcherJob do
     |> Enum.map(&(%{resource_id: &1.id} |> Transport.Jobs.GTFSRTMetadataJob.new()))
     |> Oban.insert_all()
 
+    remove_old_metadata()
+
     :ok
   end
 
+  def remove_old_metadata do
+    recent_limit = DateTime.utc_now() |> DateTime.add(-@metadata_max_nb_days, :day)
+
+    ResourceMetadata
+    |> join(:inner, [rm], r in Resource, on: rm.resource_id == r.id and r.format == "gtfs-rt")
+    |> where([rm, _r], rm.inserted_at < ^recent_limit)
+    |> select([rm, _r], rm)
+    |> Repo.delete_all()
+  end
+
   def relevant_resources do
     Resource
     |> join(:inner, [r], d in Dataset, on: r.dataset_id == d.id)
diff --git a/apps/transport/test/transport/jobs/gtfs_rt_entities_test.exs b/apps/transport/test/transport/jobs/gtfs_rt_metadata_test.exs
similarity index 76%
rename from apps/transport/test/transport/jobs/gtfs_rt_entities_test.exs
rename to apps/transport/test/transport/jobs/gtfs_rt_metadata_test.exs
index 783de8acd5..7e9e58fb64 100644
--- a/apps/transport/test/transport/jobs/gtfs_rt_entities_test.exs
+++ b/apps/transport/test/transport/jobs/gtfs_rt_metadata_test.exs
@@ -29,6 +29,20 @@ defmodule Transport.Test.Transport.Jobs.GTFSRTMetadataJobTest do
       assert :ok == perform_job(GTFSRTMetadataDispatcherJob, %{})
       assert [%Oban.Job{args: %{"resource_id" => ^resource_id}}] = all_enqueued(worker: GTFSRTMetadataJob)
     end
+
+    test "removes old metadata" do
+      resource = insert(:resource, format: "gtfs-rt")
+      gtfs_resource = insert(:resource, format: "gtfs")
+      rm1 = insert(:resource_metadata, resource_id: resource.id, inserted_at: days_ago(30))
+      rm2 = insert(:resource_metadata, resource_id: resource.id, inserted_at: days_ago(89))
+      rm3 = insert(:resource_metadata, resource_id: resource.id, inserted_at: days_ago(91))
+      rm4 = insert(:resource_metadata, resource_id: gtfs_resource.id, inserted_at: days_ago(91))
+
+      assert :ok == perform_job(GTFSRTMetadataDispatcherJob, %{})
+
+      assert [rm1, rm2, nil, rm4] == DB.Repo.reload([rm1, rm2, rm3, rm4])
+      assert resource == DB.Repo.reload(resource)
+    end
   end
 
   describe "GTFSRTMetadataJob" do
@@ -66,4 +80,8 @@ defmodule Transport.Test.Transport.Jobs.GTFSRTMetadataJobTest do
   defp setup_gtfs_rt_feed(url) do
     setup_http_response(url, {:ok, %HTTPoison.Response{status_code: 200, body: File.read!(@sample_file)}})
   end
+
+  defp days_ago(nb) when nb > 0 do
+    DateTime.utc_now() |> DateTime.add(-nb * 24 * 60 * 60)
+  end
 end