Merge pull request #259 from concord-consortium/187885602-add-glossar…

…y-data-column

feat: Add glossary data column [PT-187885602]

server/lib/report_server/post_processing/job_server.ex

@@ -6,7 +6,7 @@ defmodule ReportServer.PostProcessing.JobServer do
   alias Phoenix.PubSub
   alias ReportServerWeb.Aws
   alias ReportServer.PostProcessing.{Job, Output}
-  alias ReportServer.PostProcessing.Steps.{DemoUpperCase, DemoAddAnswerLength, HasAudio, TranscribeAudio, GlossaryAudioLinks}
+  alias ReportServer.PostProcessing.Steps.{DemoUpperCase, DemoAddAnswerLength, HasAudio, TranscribeAudio, GlossaryData}
 
   @client_check_interval :timer.minutes(1)
 
@@ -18,14 +18,14 @@ defmodule ReportServer.PostProcessing.JobServer do
     DemoUpperCase.step(),
     DemoAddAnswerLength.step(),
     HasAudio.step(),
-    GlossaryAudioLinks.step(),
+    GlossaryData.step(),
     TranscribeAudio.step()
   ] |> sort_steps()
 
   def get_steps(_mode, "details"), do: [
     HasAudio.step(),
     TranscribeAudio.step(),
-    GlossaryAudioLinks.step()
+    GlossaryData.step()
   ] |> sort_steps()
 
   def get_steps(_mode, _report_type), do: []

server/lib/report_server/post_processing/steps/glossary_data.ex

@@ -0,0 +1,85 @@
+defmodule ReportServer.PostProcessing.Steps.GlossaryData do
+  require Logger
+
+  alias ReportServer.PostProcessing.JobParams
+  alias ReportServer.PostProcessing.Step
+  alias ReportServer.PostProcessing.Steps.Helpers
+  alias ReportServerWeb.ReportService
+
+  @id "glossary_data"
+  @remote_endpoint "remote_endpoint"
+  @resource_url "resource_url"
+
+  def step do
+    %Step{
+      id: @id,
+      label: "Add glossary data column",
+      init: &init/1,
+      process_row: &process_row/3
+    }
+  end
+
+  def init(%JobParams{step_state: step_state} = params) do
+    # get all the resource columns
+    all_resource_cols = Helpers.get_resource_cols(params)
+
+    # add a column after each resource url column
+    params = Enum.reduce(all_resource_cols, params, fn {resource, resource_cols}, acc ->
+      {remote_endpoint_col, _index} = resource_cols[@remote_endpoint]
+      Helpers.add_output_column(acc, glossary_data_col(resource), :after, remote_endpoint_col)
+    end)
+
+    step_state = Map.put(step_state, @id, all_resource_cols)
+    %{params | step_state: step_state}
+  end
+
+  # header row
+  def process_row(%JobParams{step_state: step_state}, row, _data_row? = false) do
+    # copy headers to new columns
+    all_resource_cols = Map.get(step_state, @id)
+    Enum.reduce(all_resource_cols, row, fn {resource, resource_cols}, {input, output} ->
+      {_remote_endpoint_col, index} = resource_cols[@remote_endpoint]
+      {input, Map.put(output, glossary_data_col(resource), input[index])}
+    end)
+  end
+
+  # data rows
+  def process_row(%JobParams{mode: mode, step_state: step_state}, row, _data_row? = true) do
+    all_resource_cols = Map.get(step_state, @id)
+
+    Enum.reduce(all_resource_cols, row, fn {resource, resource_cols}, {input, output} ->
+      {_remote_endpoint_col, remote_endpoint_col_index} = resource_cols[@remote_endpoint]
+      {_resource_url_col, resource_url_col_index} = resource_cols[@resource_url]
+      glossary_data = case get_glossary_data(mode, input[remote_endpoint_col_index], input[resource_url_col_index]) do
+        {:ok, {key, plugin_state}} -> Jason.encode!(%{"key" => key, "plugin_state" => plugin_state})
+        _ -> ""
+      end
+      {input, Map.put(output, glossary_data_col(resource), glossary_data)}
+    end)
+  end
+
+  defp get_glossary_data(mode, remote_endpoint, resource_url) do
+    source = URI.parse(resource_url).host
+    with {:ok, plugin_states} <- ReportService.get_plugin_states(mode, source, remote_endpoint),
+         {:ok, glossary_plugin_key_and_state} <- get_first_glossary_plugin_key_and_state(plugin_states) do
+      {:ok, glossary_plugin_key_and_state}
+    else
+      {:error, error} ->
+        Logger.error(error)
+        {:error, error}
+    end
+  end
+
+  defp glossary_data_col(resource), do: "#{resource}_glossary_data"
+
+  defp get_first_glossary_plugin_key_and_state(plugin_states) do
+    first_plugin_state = Enum.find(plugin_states, fn {_k, v} ->
+      is_map(v) && Map.has_key?(v, "definitions") && is_map(v["definitions"])
+    end)
+    if first_plugin_state != nil do
+      {:ok, first_plugin_state}
+    else
+      {:error, "No glossary plugin state found"}
+    end
+  end
+end

server/lib/report_server/post_processing/steps/helpers.ex

@@ -22,6 +22,19 @@ defmodule ReportServer.PostProcessing.Steps.Helpers do
     get_cols_ending_with(params, "_remote_endpoint")
   end
 
+  def get_resource_cols(%JobParams{input_header_map: input_header_map}) do
+    Enum.reduce(input_header_map, %{}, fn {k,v}, acc ->
+      case Regex.run(~r/^(res_\d+)_(.*)/, k) do
+        [_, res, rest] ->
+          acc
+          |> Map.put_new(res, %{})
+          |> put_in([res, rest], {k,v})
+        nil ->
+          acc
+      end
+    end)
+  end
+
   def get_cols_ending_with(%JobParams{input_header_map: input_header_map}, ending) do
     Enum.reduce(input_header_map, [], fn {k,v}, acc ->
       if String.ends_with?(k, ending) do

server/lib/report_server_web/report_service.ex

@@ -74,6 +74,45 @@ defmodule ReportServerWeb.ReportService do
   def get_answer_from_response(%{"success" => false, "error" => error}), do: {:error, error}
   def get_answer_from_response(_), do: {:error, "Something went wrong getting the answer"}
 
+  # plugin states for "Test Student One" in demo data
+  def get_plugin_states("demo", "activity-player.concord.org", "https://learn.portal.staging.concord.org/dataservice/external_activity_data/36bd3464-f63a-4d0c-a7a1-aefebba85d02") do
+    {:ok, %{"demo" => %{
+      "definitions" => %{
+        "Test": ["recordingData://demo/demo-class/1", "recordingData://demo/demo-class/2", "This a test definition", "This is another test definition"],
+        "Second": ["This is the definition of second."]
+      }
+    }}}
+  end
+  def get_plugin_states("demo", _source, _remote_endpoint) do
+    # api returns an empty object when no plugin states are found
+    {:ok, %{}}
+  end
+
+  def get_plugin_states(_mode, source, remote_endpoint) do
+    with {:ok, resp} <- request_plugin_states(source, remote_endpoint),
+         {:ok, plugin_states} <- get_plugin_states_from_response(resp.body) do
+      {:ok, plugin_states}
+    else
+      {:error, error} ->
+        Logger.error("Error getting plugin states: #{error}")
+        {:error, error}
+    end
+  end
+
+  defp request_plugin_states(source, remote_endpoint) do
+    {url, token} = get_endpoint("plugin_states")
+    get_request()
+    |> Req.get(url: url,
+      auth: {:bearer, token },
+      params: [source: source, remote_endpoint: remote_endpoint],
+      debug: false
+    )
+  end
+
+  def get_plugin_states_from_response(%{"success" => true, "plugin_states" => plugin_states}), do: {:ok, plugin_states}
+  def get_plugin_states_from_response(%{"success" => false, "error" => error}), do: {:error, error}
+  def get_plugin_states_from_response(_), do: {:error, "Something went wrong getting the plugin states"}
+
   defp get_endpoint(endpoint) do
     report_service = Application.get_env(:report_server, :report_service)
     url = report_service |> Keyword.get(:url, "https://us-central1-report-service-dev.cloudfunctions.net/api")