Ajout d'une notification quand les ressources ont changé (#3347)

* Send a notification when resources changed

* Add translation

apps/transport/lib/db/notification_subscription.ex

@@ -11,7 +11,7 @@ defmodule DB.NotificationSubscription do
   @reasons_related_to_datasets [:expiration, :dataset_with_error, :resource_unavailable]
   # These notification reasons are also required to have a `dataset_id` set
   # but are not made visible to users
-  @hidden_reasons_related_to_datasets [:dataset_now_on_nap]
+  @hidden_reasons_related_to_datasets [:dataset_now_on_nap, :resources_changed]
   # These notification reasons are *not* linked to a specific dataset, `dataset_id` should be nil
   @platform_wide_reasons [:new_dataset, :datasets_switching_climate_resilience_bill, :daily_new_comments]
 
@@ -112,7 +112,8 @@ defmodule DB.NotificationSubscription do
         new_dataset: dgettext("notification_subscription", "new_dataset"),
         datasets_switching_climate_resilience_bill:
           dgettext("notification_subscription", "datasets_switching_climate_resilience_bill"),
-        daily_new_comments: dgettext("notification_subscription", "daily_new_comments")
+        daily_new_comments: dgettext("notification_subscription", "daily_new_comments"),
+        resources_changed: dgettext("notification_subscription", "resources_changed")
       },
       reason
     )

apps/transport/lib/jobs/resources_changed_notification_job.ex

@@ -0,0 +1,83 @@
+defmodule Transport.Jobs.ResourcesChangedNotificationJob do
+  @moduledoc """
+  Job in charge of detecting datasets for which resources changed:
+  - a resource has been added
+  - a resource has been deleted
+  - a download URL changed
+  and notifying subscribers about this.
+  """
+  use Oban.Worker, max_attempts: 3, tags: ["notifications"]
+  import Ecto.Query
+  @notification_reason :resources_changed
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: args}) when is_nil(args) or args == %{} do
+    relevant_datasets()
+    |> Enum.map(&new/1)
+    |> Oban.insert_all()
+
+    :ok
+  end
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{args: %{"dataset_id" => dataset_id}}) do
+    dataset = DB.Dataset |> DB.Repo.get!(dataset_id)
+    subject = "#{dataset.custom_title} : ressources modifiées"
+
+    @notification_reason
+    |> DB.NotificationSubscription.subscriptions_for_reason()
+    |> DB.NotificationSubscription.subscriptions_to_emails()
+    |> Enum.each(fn email ->
+      Transport.EmailSender.impl().send_mail(
+        "transport.data.gouv.fr",
+        Application.get_env(:transport, :contact_email),
+        email,
+        Application.get_env(:transport, :contact_email),
+        subject,
+        "",
+        Phoenix.View.render_to_string(TransportWeb.EmailView, "resources_changed.html", dataset: dataset)
+      )
+
+      save_notification(dataset, email)
+    end)
+  end
+
+  def save_notification(%DB.Dataset{} = dataset, email) do
+    DB.Notification.insert!(@notification_reason, dataset, email)
+  end
+
+  def relevant_datasets do
+    today = Date.utc_today()
+
+    # Latest `dataset_history` by dataset by day
+    # (usually 1 row per day but make sure duplicates are handled)
+    history_by_day_sub =
+      DB.DatasetHistory
+      |> select([d], max(d.id))
+      |> group_by([d], [d.dataset_id, fragment("?::date", d.inserted_at)])
+
+    urls_by_day_sub =
+      DB.DatasetHistory
+      |> join(:inner, [dh], dhr in DB.DatasetHistoryResources, on: dh.id == dhr.dataset_history_id)
+      |> join(:inner, [_dh, dhr], r in DB.Resource, on: r.id == dhr.resource_id and not r.is_community_resource)
+      |> where([dh, _dhr, _r], dh.id in subquery(history_by_day_sub))
+      |> group_by([dh, _dhr, _r], [dh.dataset_id, fragment("?::date", dh.inserted_at)])
+      |> select([dh, dhr, _d], %{
+        dataset_id: dh.dataset_id,
+        date: fragment("?::date", dh.inserted_at),
+        urls: fragment("string_agg(?->>'download_url', ',' order by ?->>'download_url')", dhr.payload, dhr.payload)
+      })
+
+    base = from(today in subquery(urls_by_day_sub))
+
+    base
+    |> join(:inner, [today], yesterday in subquery(urls_by_day_sub),
+      on:
+        today.dataset_id == yesterday.dataset_id and
+          yesterday.date == fragment("? - 1", today.date) and
+          today.urls != yesterday.urls
+    )
+    |> where([today, _yesterday], today.date == ^today)
+    |> DB.Repo.all()
+  end
+end

apps/transport/lib/transport_web/templates/email/resources_changed.html.md

@@ -0,0 +1,5 @@
+Bonjour,
+
+Les ressources du jeu de données <%= link_for_dataset(@dataset, :heex) %> viennent d'être modifiées (ajout/suppression de ressources ou modification d'URLs de téléchargement).
+
+L’équipe du PAN

apps/transport/priv/gettext/en/LC_MESSAGES/notification_subscription.po

@@ -38,3 +38,7 @@ msgstr "Unavailable resources"
 #, elixir-autogen, elixir-format
 msgid "dataset_now_on_nap"
 msgstr "Dataset has been added to the NAP"
+
+#, elixir-autogen, elixir-format
+msgid "resources_changed"
+msgstr "Dataset resources changed"

apps/transport/priv/gettext/fr/LC_MESSAGES/notification_subscription.po

@@ -38,3 +38,7 @@ msgstr "Ressources indisponibles"
 #, elixir-autogen, elixir-format
 msgid "dataset_now_on_nap"
 msgstr "Le jeu de données a été référencé sur le PAN"
+
+#, elixir-autogen, elixir-format
+msgid "resources_changed"
+msgstr "Les ressources d'un jeu de données ont été modifiées"

apps/transport/priv/gettext/notification_subscription.pot

@@ -38,3 +38,7 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "dataset_now_on_nap"
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "resources_changed"
+msgstr ""

apps/transport/test/transport/jobs/resources_changed_notification_job_test.exs

@@ -0,0 +1,171 @@
+defmodule Transport.Test.Transport.Jobs.ResourcesChangedNotificationJobTest do
+  use ExUnit.Case, async: true
+  import DB.Factory
+  import Mox
+  use Oban.Testing, repo: DB.Repo
+  alias Transport.Jobs.ResourcesChangedNotificationJob
+
+  setup do
+    Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo)
+  end
+
+  setup :verify_on_exit!
+
+  test "relevant_datasets and dispatches jobs" do
+    today = DateTime.utc_now()
+    today_date = today |> DateTime.to_date()
+    yesterday = today |> DateTime.add(-1, :day)
+    day_before = today |> DateTime.add(-2, :day)
+
+    %DB.Dataset{id: dataset_id} = dataset = insert(:dataset)
+    other_dataset = insert(:dataset)
+
+    resource_dataset_gtfs =
+      insert(:resource,
+        dataset: dataset,
+        format: "GTFS",
+        url: "https://example.com/gtfs.zip",
+        is_community_resource: false
+      )
+
+    resource_dataset_gtfs_rt =
+      insert(:resource,
+        dataset: dataset,
+        format: "gtfs-rt",
+        url: "https://example.com/gtfs-rt",
+        is_community_resource: false
+      )
+
+    resource_other_dataset_gtfs =
+      insert(:resource,
+        dataset: other_dataset,
+        format: "GTFS",
+        is_community_resource: false,
+        url: "https://example.fr/gtfs.zip"
+      )
+
+    community_resource_other_dataset =
+      insert(:resource,
+        dataset: other_dataset,
+        format: "GTFS",
+        is_community_resource: true,
+        url: "https://lemonde.fr/pan"
+      )
+
+    resource_other_dataset_gtfs_rt =
+      insert(:resource,
+        dataset: other_dataset,
+        format: "gtfs-rt",
+        is_community_resource: false,
+        url: "https://example.fr/gtfs.zip"
+      )
+
+    # `dataset` has a new resource today, `other_dataset` did not change today but before
+    dataset_dh_today = insert(:dataset_history, dataset: dataset, inserted_at: today)
+    dataset_dh_yesterday = insert(:dataset_history, dataset: dataset, inserted_at: yesterday)
+    other_dataset_dh_today = insert(:dataset_history, dataset: other_dataset, inserted_at: today)
+    other_dataset_dh_yesterday = insert(:dataset_history, dataset: other_dataset, inserted_at: yesterday)
+    other_dataset_dh_day_before = insert(:dataset_history, dataset: other_dataset, inserted_at: day_before)
+
+    # For `dataset`
+    # Today: GTFS-RT + GTFS
+    # Yesterday: GTFS
+    insert(:dataset_history_resources,
+      resource: resource_dataset_gtfs,
+      payload: %{"download_url" => resource_dataset_gtfs.url},
+      dataset_history: dataset_dh_today
+    )
+
+    insert(:dataset_history_resources,
+      resource: resource_dataset_gtfs_rt,
+      payload: %{"download_url" => resource_dataset_gtfs_rt.url},
+      dataset_history: dataset_dh_today
+    )
+
+    insert(:dataset_history_resources,
+      resource: resource_dataset_gtfs,
+      payload: %{"download_url" => resource_dataset_gtfs.url},
+      dataset_history: dataset_dh_yesterday
+    )
+
+    # For `other_dataset`
+    # - 2 days ago: GTFS-RT + GTFS (should be ignored)
+    # - Only a GTFS for yesterday and today
+    # - A new community resource was added today (should be ignored)
+    insert(:dataset_history_resources,
+      resource: resource_other_dataset_gtfs,
+      payload: %{"download_url" => resource_other_dataset_gtfs.url},
+      dataset_history: other_dataset_dh_today
+    )
+
+    insert(:dataset_history_resources,
+      resource: community_resource_other_dataset,
+      payload: %{"download_url" => community_resource_other_dataset.url},
+      dataset_history: other_dataset_dh_today
+    )
+
+    insert(:dataset_history_resources,
+      resource: resource_other_dataset_gtfs,
+      payload: %{"download_url" => resource_other_dataset_gtfs.url},
+      dataset_history: other_dataset_dh_yesterday
+    )
+
+    insert(:dataset_history_resources,
+      resource: resource_other_dataset_gtfs,
+      payload: %{"download_url" => resource_other_dataset_gtfs.url},
+      dataset_history: other_dataset_dh_day_before
+    )
+
+    insert(:dataset_history_resources,
+      resource: resource_other_dataset_gtfs_rt,
+      payload: %{"download_url" => resource_other_dataset_gtfs_rt.url},
+      dataset_history: other_dataset_dh_day_before
+    )
+
+    assert [
+             %{
+               dataset_id: ^dataset_id,
+               date: ^today_date,
+               urls: "https://example.com/gtfs-rt,https://example.com/gtfs.zip"
+             }
+           ] = ResourcesChangedNotificationJob.relevant_datasets()
+
+    assert :ok == perform_job(ResourcesChangedNotificationJob, %{})
+
+    assert [%Oban.Job{worker: "Transport.Jobs.ResourcesChangedNotificationJob", args: %{"dataset_id" => ^dataset_id}}] =
+             all_enqueued()
+  end
+
+  test "perform with a dataset_id" do
+    %DB.Dataset{id: dataset_id} = dataset = insert(:dataset, custom_title: "Super JDD")
+    %DB.Contact{id: contact_id, email: email} = insert_contact()
+
+    insert(:notification_subscription, %{
+      reason: :resources_changed,
+      source: :admin,
+      role: :reuser,
+      contact_id: contact_id
+    })
+
+    Transport.EmailSender.Mock
+    |> expect(:send_mail, fn "transport.data.gouv.fr",
+                             "[email protected]",
+                             ^email,
+                             "[email protected]",
+                             "Super JDD : ressources modifiées" = _subject,
+                             "",
+                             html_part ->
+      assert html_part =~
+               ~s(Les ressources du jeu de données <a href="http://127.0.0.1:5100/datasets/#{dataset.slug}">#{dataset.custom_title}</a> viennent d’être modifiées)
+
+      :ok
+    end)
+
+    assert :ok == perform_job(ResourcesChangedNotificationJob, %{"dataset_id" => dataset_id})
+
+    # Logs have been saved
+    assert [
+             %DB.Notification{email: ^email, reason: :resources_changed, dataset_id: ^dataset_id}
+           ] = DB.Notification |> DB.Repo.all()
+  end
+end