From 10ea7e067f0a85c1bc7c89bf19934090fc2c6fc7 Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Tue, 25 Jul 2023 18:15:07 +0200 Subject: [PATCH] =?UTF-8?q?Ajout=20d'une=20notification=20quand=20les=20re?= =?UTF-8?q?ssources=20ont=20chang=C3=A9=20(#3347)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Send a notification when resources changed * Add translation --- .../lib/db/notification_subscription.ex | 5 +- .../resources_changed_notification_job.ex | 83 +++++++++ .../templates/email/resources_changed.html.md | 5 + .../LC_MESSAGES/notification_subscription.po | 4 + .../LC_MESSAGES/notification_subscription.po | 4 + .../gettext/notification_subscription.pot | 4 + ...esources_changed_notification_job_test.exs | 171 ++++++++++++++++++ 7 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 apps/transport/lib/jobs/resources_changed_notification_job.ex create mode 100644 apps/transport/lib/transport_web/templates/email/resources_changed.html.md create mode 100644 apps/transport/test/transport/jobs/resources_changed_notification_job_test.exs diff --git a/apps/transport/lib/db/notification_subscription.ex b/apps/transport/lib/db/notification_subscription.ex index 9d576aa8af..3dabcbc87e 100644 --- a/apps/transport/lib/db/notification_subscription.ex +++ b/apps/transport/lib/db/notification_subscription.ex @@ -11,7 +11,7 @@ defmodule DB.NotificationSubscription do @reasons_related_to_datasets [:expiration, :dataset_with_error, :resource_unavailable] # These notification reasons are also required to have a `dataset_id` set # but are not made visible to users - @hidden_reasons_related_to_datasets [:dataset_now_on_nap] + @hidden_reasons_related_to_datasets [:dataset_now_on_nap, :resources_changed] # These notification reasons are *not* linked to a specific dataset, `dataset_id` should be nil @platform_wide_reasons [:new_dataset, :datasets_switching_climate_resilience_bill, :daily_new_comments] @@ -112,7 +112,8 @@ defmodule DB.NotificationSubscription do new_dataset: dgettext("notification_subscription", "new_dataset"), datasets_switching_climate_resilience_bill: dgettext("notification_subscription", "datasets_switching_climate_resilience_bill"), - daily_new_comments: dgettext("notification_subscription", "daily_new_comments") + daily_new_comments: dgettext("notification_subscription", "daily_new_comments"), + resources_changed: dgettext("notification_subscription", "resources_changed") }, reason ) diff --git a/apps/transport/lib/jobs/resources_changed_notification_job.ex b/apps/transport/lib/jobs/resources_changed_notification_job.ex new file mode 100644 index 0000000000..69919b0272 --- /dev/null +++ b/apps/transport/lib/jobs/resources_changed_notification_job.ex @@ -0,0 +1,83 @@ +defmodule Transport.Jobs.ResourcesChangedNotificationJob do + @moduledoc """ + Job in charge of detecting datasets for which resources changed: + - a resource has been added + - a resource has been deleted + - a download URL changed + and notifying subscribers about this. + """ + use Oban.Worker, max_attempts: 3, tags: ["notifications"] + import Ecto.Query + @notification_reason :resources_changed + + @impl Oban.Worker + def perform(%Oban.Job{args: args}) when is_nil(args) or args == %{} do + relevant_datasets() + |> Enum.map(&new/1) + |> Oban.insert_all() + + :ok + end + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"dataset_id" => dataset_id}}) do + dataset = DB.Dataset |> DB.Repo.get!(dataset_id) + subject = "#{dataset.custom_title} : ressources modifiées" + + @notification_reason + |> DB.NotificationSubscription.subscriptions_for_reason() + |> DB.NotificationSubscription.subscriptions_to_emails() + |> Enum.each(fn email -> + Transport.EmailSender.impl().send_mail( + "transport.data.gouv.fr", + Application.get_env(:transport, :contact_email), + email, + Application.get_env(:transport, :contact_email), + subject, + "", + Phoenix.View.render_to_string(TransportWeb.EmailView, "resources_changed.html", dataset: dataset) + ) + + save_notification(dataset, email) + end) + end + + def save_notification(%DB.Dataset{} = dataset, email) do + DB.Notification.insert!(@notification_reason, dataset, email) + end + + def relevant_datasets do + today = Date.utc_today() + + # Latest `dataset_history` by dataset by day + # (usually 1 row per day but make sure duplicates are handled) + history_by_day_sub = + DB.DatasetHistory + |> select([d], max(d.id)) + |> group_by([d], [d.dataset_id, fragment("?::date", d.inserted_at)]) + + urls_by_day_sub = + DB.DatasetHistory + |> join(:inner, [dh], dhr in DB.DatasetHistoryResources, on: dh.id == dhr.dataset_history_id) + |> join(:inner, [_dh, dhr], r in DB.Resource, on: r.id == dhr.resource_id and not r.is_community_resource) + |> where([dh, _dhr, _r], dh.id in subquery(history_by_day_sub)) + |> group_by([dh, _dhr, _r], [dh.dataset_id, fragment("?::date", dh.inserted_at)]) + |> select([dh, dhr, _d], %{ + dataset_id: dh.dataset_id, + date: fragment("?::date", dh.inserted_at), + urls: fragment("string_agg(?->>'download_url', ',' order by ?->>'download_url')", dhr.payload, dhr.payload) + }) + + base = from(today in subquery(urls_by_day_sub)) + + base + |> join(:inner, [today], yesterday in subquery(urls_by_day_sub), + on: + today.dataset_id == yesterday.dataset_id and + yesterday.date == fragment("? - 1", today.date) and + today.urls != yesterday.urls + ) + |> where([today, _yesterday], today.date == ^today) + |> DB.Repo.all() + end +end diff --git a/apps/transport/lib/transport_web/templates/email/resources_changed.html.md b/apps/transport/lib/transport_web/templates/email/resources_changed.html.md new file mode 100644 index 0000000000..82fabc6623 --- /dev/null +++ b/apps/transport/lib/transport_web/templates/email/resources_changed.html.md @@ -0,0 +1,5 @@ +Bonjour, + +Les ressources du jeu de données <%= link_for_dataset(@dataset, :heex) %> viennent d'être modifiées (ajout/suppression de ressources ou modification d'URLs de téléchargement). + +L’équipe du PAN diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/notification_subscription.po b/apps/transport/priv/gettext/en/LC_MESSAGES/notification_subscription.po index 7219935d7c..cbd3690503 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/notification_subscription.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/notification_subscription.po @@ -38,3 +38,7 @@ msgstr "Unavailable resources" #, elixir-autogen, elixir-format msgid "dataset_now_on_nap" msgstr "Dataset has been added to the NAP" + +#, elixir-autogen, elixir-format +msgid "resources_changed" +msgstr "Dataset resources changed" diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/notification_subscription.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/notification_subscription.po index 0dc577e9e3..4ecfaf246f 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/notification_subscription.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/notification_subscription.po @@ -38,3 +38,7 @@ msgstr "Ressources indisponibles" #, elixir-autogen, elixir-format msgid "dataset_now_on_nap" msgstr "Le jeu de données a été référencé sur le PAN" + +#, elixir-autogen, elixir-format +msgid "resources_changed" +msgstr "Les ressources d'un jeu de données ont été modifiées" diff --git a/apps/transport/priv/gettext/notification_subscription.pot b/apps/transport/priv/gettext/notification_subscription.pot index 1ec50bf5d7..a43a1c5f32 100644 --- a/apps/transport/priv/gettext/notification_subscription.pot +++ b/apps/transport/priv/gettext/notification_subscription.pot @@ -38,3 +38,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "dataset_now_on_nap" msgstr "" + +#, elixir-autogen, elixir-format +msgid "resources_changed" +msgstr "" diff --git a/apps/transport/test/transport/jobs/resources_changed_notification_job_test.exs b/apps/transport/test/transport/jobs/resources_changed_notification_job_test.exs new file mode 100644 index 0000000000..45ca9721cd --- /dev/null +++ b/apps/transport/test/transport/jobs/resources_changed_notification_job_test.exs @@ -0,0 +1,171 @@ +defmodule Transport.Test.Transport.Jobs.ResourcesChangedNotificationJobTest do + use ExUnit.Case, async: true + import DB.Factory + import Mox + use Oban.Testing, repo: DB.Repo + alias Transport.Jobs.ResourcesChangedNotificationJob + + setup do + Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + setup :verify_on_exit! + + test "relevant_datasets and dispatches jobs" do + today = DateTime.utc_now() + today_date = today |> DateTime.to_date() + yesterday = today |> DateTime.add(-1, :day) + day_before = today |> DateTime.add(-2, :day) + + %DB.Dataset{id: dataset_id} = dataset = insert(:dataset) + other_dataset = insert(:dataset) + + resource_dataset_gtfs = + insert(:resource, + dataset: dataset, + format: "GTFS", + url: "https://example.com/gtfs.zip", + is_community_resource: false + ) + + resource_dataset_gtfs_rt = + insert(:resource, + dataset: dataset, + format: "gtfs-rt", + url: "https://example.com/gtfs-rt", + is_community_resource: false + ) + + resource_other_dataset_gtfs = + insert(:resource, + dataset: other_dataset, + format: "GTFS", + is_community_resource: false, + url: "https://example.fr/gtfs.zip" + ) + + community_resource_other_dataset = + insert(:resource, + dataset: other_dataset, + format: "GTFS", + is_community_resource: true, + url: "https://lemonde.fr/pan" + ) + + resource_other_dataset_gtfs_rt = + insert(:resource, + dataset: other_dataset, + format: "gtfs-rt", + is_community_resource: false, + url: "https://example.fr/gtfs.zip" + ) + + # `dataset` has a new resource today, `other_dataset` did not change today but before + dataset_dh_today = insert(:dataset_history, dataset: dataset, inserted_at: today) + dataset_dh_yesterday = insert(:dataset_history, dataset: dataset, inserted_at: yesterday) + other_dataset_dh_today = insert(:dataset_history, dataset: other_dataset, inserted_at: today) + other_dataset_dh_yesterday = insert(:dataset_history, dataset: other_dataset, inserted_at: yesterday) + other_dataset_dh_day_before = insert(:dataset_history, dataset: other_dataset, inserted_at: day_before) + + # For `dataset` + # Today: GTFS-RT + GTFS + # Yesterday: GTFS + insert(:dataset_history_resources, + resource: resource_dataset_gtfs, + payload: %{"download_url" => resource_dataset_gtfs.url}, + dataset_history: dataset_dh_today + ) + + insert(:dataset_history_resources, + resource: resource_dataset_gtfs_rt, + payload: %{"download_url" => resource_dataset_gtfs_rt.url}, + dataset_history: dataset_dh_today + ) + + insert(:dataset_history_resources, + resource: resource_dataset_gtfs, + payload: %{"download_url" => resource_dataset_gtfs.url}, + dataset_history: dataset_dh_yesterday + ) + + # For `other_dataset` + # - 2 days ago: GTFS-RT + GTFS (should be ignored) + # - Only a GTFS for yesterday and today + # - A new community resource was added today (should be ignored) + insert(:dataset_history_resources, + resource: resource_other_dataset_gtfs, + payload: %{"download_url" => resource_other_dataset_gtfs.url}, + dataset_history: other_dataset_dh_today + ) + + insert(:dataset_history_resources, + resource: community_resource_other_dataset, + payload: %{"download_url" => community_resource_other_dataset.url}, + dataset_history: other_dataset_dh_today + ) + + insert(:dataset_history_resources, + resource: resource_other_dataset_gtfs, + payload: %{"download_url" => resource_other_dataset_gtfs.url}, + dataset_history: other_dataset_dh_yesterday + ) + + insert(:dataset_history_resources, + resource: resource_other_dataset_gtfs, + payload: %{"download_url" => resource_other_dataset_gtfs.url}, + dataset_history: other_dataset_dh_day_before + ) + + insert(:dataset_history_resources, + resource: resource_other_dataset_gtfs_rt, + payload: %{"download_url" => resource_other_dataset_gtfs_rt.url}, + dataset_history: other_dataset_dh_day_before + ) + + assert [ + %{ + dataset_id: ^dataset_id, + date: ^today_date, + urls: "https://example.com/gtfs-rt,https://example.com/gtfs.zip" + } + ] = ResourcesChangedNotificationJob.relevant_datasets() + + assert :ok == perform_job(ResourcesChangedNotificationJob, %{}) + + assert [%Oban.Job{worker: "Transport.Jobs.ResourcesChangedNotificationJob", args: %{"dataset_id" => ^dataset_id}}] = + all_enqueued() + end + + test "perform with a dataset_id" do + %DB.Dataset{id: dataset_id} = dataset = insert(:dataset, custom_title: "Super JDD") + %DB.Contact{id: contact_id, email: email} = insert_contact() + + insert(:notification_subscription, %{ + reason: :resources_changed, + source: :admin, + role: :reuser, + contact_id: contact_id + }) + + Transport.EmailSender.Mock + |> expect(:send_mail, fn "transport.data.gouv.fr", + "contact@transport.beta.gouv.fr", + ^email, + "contact@transport.beta.gouv.fr", + "Super JDD : ressources modifiées" = _subject, + "", + html_part -> + assert html_part =~ + ~s(Les ressources du jeu de données #{dataset.custom_title} viennent d’être modifiées) + + :ok + end) + + assert :ok == perform_job(ResourcesChangedNotificationJob, %{"dataset_id" => dataset_id}) + + # Logs have been saved + assert [ + %DB.Notification{email: ^email, reason: :resources_changed, dataset_id: ^dataset_id} + ] = DB.Notification |> DB.Repo.all() + end +end