Affichage d'un graphique pour les scores d'un JDD (#3372)

* Affichage d'un graphique pour les scores d'un JDD

* Remove semicolons

apps/transport/client/javascripts/vega.js

@@ -0,0 +1,3 @@
+import embed from 'vega-embed'
+
+window.vegaEmbed = embed

apps/transport/client/package.json

@@ -28,6 +28,9 @@
     "phoenix_live_view": "file:../../../deps/phoenix_live_view",
     "prismjs": "^1.29.0",
     "template.data.gouv.fr": "^1.2.1",
+    "vega": "^5.25.0",
+    "vega-embed": "^6.22.2",
+    "vega-lite": "^5.14.1",
     "xml-formatter": "^2.6.1"
   },
   "devDependencies": {

apps/transport/client/stylesheets/components/_dataset-details.scss

@@ -597,3 +597,10 @@ hr {
     margin-bottom: 1em;
   }
 }
+
+#vega-vis {
+  width: 100%;
+  max-width: 900px;
+  margin: 1em auto;
+  display: block;
+}

apps/transport/client/webpack.common.js

@@ -11,6 +11,7 @@ module.exports = {
         app: './javascripts/app.js',
         clipboard: './javascripts/clipboard.js',
         map: './javascripts/map.js',
+        vega: './javascripts/vega.js',
         resourceviz: './javascripts/resource-viz.js',
         explore: './javascripts/explore.js',
         gtfs: './javascripts/gtfs.js',

apps/transport/lib/db/dataset_score.ex

@@ -59,4 +59,25 @@ defmodule DB.DatasetScore do
     |> DB.Repo.all()
     |> Enum.into(%{}, fn %__MODULE__{topic: topic} = ds -> {topic, ds} end)
   end
+
+  def scores_over_last_days(%DB.Dataset{id: dataset_id}, nb_days) when is_integer(nb_days) and nb_days > 0 do
+    dt_limit = %DateTime{DateTime.utc_now() | hour: 0, minute: 0, second: 0} |> DateTime.add(-nb_days, :day)
+
+    # Find the latest point for each topic for this dataset, for each day
+    ids =
+      __MODULE__.base_query()
+      |> where([dataset_score: ds], ds.dataset_id == ^dataset_id and ds.timestamp >= ^dt_limit)
+      |> select(
+        [dataset_score: ds],
+        ds.id
+        |> first_value()
+        |> over(partition_by: [ds.topic, fragment("?::date", ds.timestamp)], order_by: [desc: ds.timestamp])
+      )
+      |> distinct(true)
+
+    __MODULE__.base_query()
+    |> where([dataset_score: ds], ds.id in subquery(ids))
+    |> order_by([dataset_score: ds], asc: ds.timestamp, asc: ds.topic)
+    |> DB.Repo.all()
+  end
 end

apps/transport/lib/transport_web/controllers/dataset_controller.ex

@@ -60,6 +60,7 @@ defmodule TransportWeb.DatasetController do
       |> assign(:latest_resources_history_infos, DB.ResourceHistory.latest_dataset_resources_history_infos(dataset.id))
       |> assign(:notifications_sent, DB.Notification.recent_reasons_binned(dataset, days_notifications_sent()))
       |> assign(:dataset_scores, DB.DatasetScore.get_latest_scores(dataset, Ecto.Enum.values(DB.DatasetScore, :topic)))
+      |> assign(:scores_chart, scores_chart(dataset))
       |> put_status(if dataset.is_active, do: :ok, else: :not_found)
       |> render("details.html")
     else
@@ -72,6 +73,26 @@ defmodule TransportWeb.DatasetController do
     end
   end
 
+  def scores_chart(%DB.Dataset{} = dataset) do
+    data = DB.DatasetScore.scores_over_last_days(dataset, 30 * 3)
+
+    # See https://hexdocs.pm/vega_lite/
+    # and https://vega.github.io/vega-lite/docs/
+    [width: "container", height: 250]
+    |> VegaLite.new()
+    |> VegaLite.data_from_values(
+      Enum.map(data, fn %DB.DatasetScore{} = ds ->
+        %{"topic" => ds.topic, "score" => ds.score, "date" => ds.timestamp |> DateTime.to_date()}
+      end)
+    )
+    |> VegaLite.mark(:line, interpolate: "step-before", tooltip: true, strokeWidth: 3)
+    |> VegaLite.encode_field(:x, "date", type: :temporal)
+    |> VegaLite.encode_field(:y, "score", type: :quantitative)
+    |> VegaLite.encode_field(:color, "topic", type: :nominal)
+    |> VegaLite.config(axis: [grid: false])
+    |> VegaLite.to_spec()
+  end
+
   def validators_to_use,
     do: [
       Transport.Validators.GTFSTransport,

apps/transport/lib/transport_web/templates/dataset/_dataset_resources_history.html.heex

@@ -1,5 +1,5 @@
 <% has_validity_period_col = has_validity_period?(@history_resources) %>
-<section class="white pt-48" id="backed-up-resources">
+<section :if={Enum.count(@history_resources) > 0} class="white pt-48" id="backed-up-resources">
   <h3><%= dgettext("page-dataset-details", "Backed up resources") %></h3>
   <div class="panel">
     <div id="backed-up-resources-see-more-wrapper">

apps/transport/lib/transport_web/templates/dataset/_dataset_scores.html.heex

@@ -21,4 +21,5 @@
       Pas de score de disponibilité
     <% end %>
   </div>
+  <a href="#scores-chart">Voir plus</a>
 </div>

apps/transport/lib/transport_web/templates/dataset/_dataset_scores_chart.html.heex

@@ -0,0 +1,12 @@
+<section :if={admin?(@conn.assigns[:current_user])} class="pt-48">
+  <h2><%= dgettext("page-dataset-details", "Dataset scores") %></h2>
+  <div class="panel" id="scores-chart">
+    <div id="vega-vis"></div>
+    <p class="small">Ceci est visible uniquement par les membres de transport.data.gouv.fr.</p>
+    <script src={static_path(@conn, "/js/vega.js")} />
+    <script>
+      const spec = <%= raw Jason.encode!(@scores_chart) %>;
+      window.vegaEmbed("#vega-vis", spec, {renderer: "svg"});
+    </script>
+  </div>
+</section>

apps/transport/lib/transport_web/templates/dataset/details.html.heex

@@ -220,9 +220,8 @@
         </div>
       </section>
     <% end %>
-    <%= unless @history_resources == [] do %>
-      <%= render("_dataset_resources_history.html", history_resources: @history_resources, locale: locale, conn: @conn) %>
-    <% end %>
+    <%= render("_dataset_scores_chart.html", scores_chart: @scores_chart, conn: @conn) %>
+    <%= render("_dataset_resources_history.html", history_resources: @history_resources, locale: locale, conn: @conn) %>
     <%= unless is_nil(@other_datasets) or @other_datasets == [] do %>
       <section class="pt-48" id="dataset-other-datasets">
         <h2><%= dgettext("page-dataset-details", "Other datasets of %{name}", name: @territory) %></h2>

apps/transport/mix.exs

@@ -129,8 +129,8 @@ defmodule Transport.Mixfile do
       {:luhn, "~> 0.3.0"},
       {:ex_phone_number, "~> 0.3"},
       {:appsignal, "~> 2.0"},
-      {:appsignal_phoenix, "~> 2.0"}
-
+      {:appsignal_phoenix, "~> 2.0"},
+      {:vega_lite, "~> 0.1.7"}
     ]
   end
 end

apps/transport/priv/gettext/en/LC_MESSAGES/page-dataset-details.po

@@ -601,3 +601,7 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "loading discussions..."
 msgstr ""
+
+#, elixir-autogen, elixir-format, fuzzy
+msgid "Dataset scores"
+msgstr ""

apps/transport/priv/gettext/fr/LC_MESSAGES/page-dataset-details.po

@@ -601,3 +601,7 @@ msgstr "transport.data.gouv.fr envoie automatiquement des notifications au produ
 #, elixir-autogen, elixir-format
 msgid "loading discussions..."
 msgstr "chargement des discussions..."
+
+#, elixir-autogen, elixir-format
+msgid "Dataset scores"
+msgstr "Scores du jeu de données"

apps/transport/priv/gettext/page-dataset-details.pot

@@ -601,3 +601,7 @@ msgstr ""
 #, elixir-autogen, elixir-format
 msgid "loading discussions..."
 msgstr ""
+
+#, elixir-autogen, elixir-format
+msgid "Dataset scores"
+msgstr ""

apps/transport/test/db/dataset_score_test.exs

@@ -115,4 +115,46 @@ defmodule DB.DatasetScoreTest do
 
     assert %{} == DB.DatasetScore.get_latest_scores(%DB.Dataset{id: 123_456}, [:freshness, :availability])
   end
+
+  test "scores_over_last_days" do
+    dataset = insert(:dataset)
+    other_dataset = insert(:dataset)
+
+    # other dataset
+    insert(:dataset_score, dataset: other_dataset, timestamp: DateTime.utc_now(), score: 1.0, topic: :freshness)
+
+    [{-3, 0.75}, {-2, 0.5}, {-1, 0.75}, {0, 1}]
+    |> Enum.each(fn {days_delta, score} ->
+      insert(:dataset_score,
+        dataset: dataset,
+        timestamp: DateTime.utc_now() |> DateTime.add(days_delta, :day),
+        score: score,
+        topic: :freshness
+      )
+
+      insert(:dataset_score,
+        dataset: dataset,
+        timestamp: DateTime.utc_now() |> DateTime.add(days_delta, :day),
+        score: score - 0.25,
+        topic: :availability
+      )
+    end)
+
+    # should be ignored, not the latest point for today
+    insert(:dataset_score,
+      dataset: dataset,
+      timestamp: DateTime.utc_now() |> DateTime.add(-5, :minute),
+      score: 0,
+      topic: :freshness
+    )
+
+    assert [
+             %DB.DatasetScore{topic: :freshness, score: 0.5},
+             %DB.DatasetScore{topic: :availability, score: 0.25},
+             %DB.DatasetScore{topic: :freshness, score: 0.75},
+             %DB.DatasetScore{topic: :availability, score: 0.5},
+             %DB.DatasetScore{topic: :freshness, score: 1.0},
+             %DB.DatasetScore{topic: :availability, score: 0.75}
+           ] = DB.DatasetScore.scores_over_last_days(dataset, 2)
+  end
 end

apps/transport/test/transport_web/controllers/dataset_controller_test.exs

@@ -396,6 +396,33 @@ defmodule TransportWeb.DatasetControllerTest do
     refute content =~ "Score fraicheur"
   end
 
+  describe "scores_chart" do
+    test "is displayed for admins", %{conn: conn} do
+      dataset = insert(:dataset, is_active: true)
+      set_empty_mocks()
+
+      refute conn
+             |> setup_admin_in_session()
+             |> get(dataset_path(conn, :details, dataset.slug))
+             |> html_response(200)
+             |> Floki.parse_document!()
+             |> Floki.find("#scores-chart")
+             |> Enum.empty?()
+    end
+
+    test "is not displayed for regular users", %{conn: conn} do
+      dataset = insert(:dataset, is_active: true)
+      set_empty_mocks()
+
+      assert conn
+             |> get(dataset_path(conn, :details, dataset.slug))
+             |> html_response(200)
+             |> Floki.parse_document!()
+             |> Floki.find("#scores-chart")
+             |> Enum.empty?()
+    end
+  end
+
   test "gtfs-rt entities" do
     dataset = %{id: dataset_id} = insert(:dataset, type: "public-transit")
     %{id: resource_id_1} = insert(:resource, dataset_id: dataset_id, format: "gtfs-rt")

mix.lock

@@ -125,6 +125,7 @@
   "unidecode": {:hex, :unidecode, "1.0.1", "d653e7e9777b7c1bcf48b923bbbb9167ecd21a2a90aabb5a3c65b32c6a359497", [:mix], [], "hexpm", "73b490342603a5b1083e0f8f8b9e9bee4149ada327d1b9cdbaf8259f04699611"},
   "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"},
   "unzip": {:hex, :unzip, "0.8.0", "ee21d87c21b01567317387dab4228ac570ca15b41cfc221a067354cbf8e68c4d", [:mix], [], "hexpm", "ffa67a483efcedcb5876971a50947222e104d5f8fea2c4a0441e6f7967854827"},
+  "vega_lite": {:hex, :vega_lite, "0.1.7", "1f1ad850821335702958f7351ca52808a5a109f9ce35fd5c14f9613970bad140", [:mix], [{:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: false]}], "hexpm", "3932382ed5ad77c08c27abc45137257e9a86897916cef1a0da148632c6a91e30"},
   "vex": {:git, "https://github.com/CargoSense/vex.git", "328a39f7688000e2746386eabe2f40e6c8e7796b", [ref: "328a39f7"]},
   "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
   "yaml_elixir": {:hex, :yaml_elixir, "2.9.0", "9a256da867b37b8d2c1ffd5d9de373a4fda77a32a45b452f1708508ba7bbcb53", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "0cb0e7d4c56f5e99a6253ed1a670ed0e39c13fc45a6da054033928607ac08dfc"},