From e7b42179d3c14c2cfe7fc0f2e1108c9374a3f927 Mon Sep 17 00:00:00 2001
From: "Michael B. Klein" <mbklein@gmail.com>
Date: Tue, 5 Nov 2024 22:22:29 +0000
Subject: [PATCH] Push more test configs into localstack secrets manager

---
 app/lib/meadow/config/runtime.ex              |  87 +----------
 app/lib/meadow/config/runtime/dev.ex          |   2 +-
 app/lib/meadow/config/runtime/test.ex         |  41 +-----
 app/lib/meadow/config/secrets.ex              |  91 ++++++++++++
 app/lib/meadow/utils/aws.ex                   |   4 +-
 .../localstack/environment_config.tf          | 137 +++++++++++++-----
 infrastructure/localstack/test.tfvars         |  25 ----
 infrastructure/localstack/variables.tf        |   5 -
 8 files changed, 196 insertions(+), 196 deletions(-)
 create mode 100644 app/lib/meadow/config/secrets.ex

diff --git a/app/lib/meadow/config/runtime.ex b/app/lib/meadow/config/runtime.ex
index 362a8b4e4..50ffb7334 100644
--- a/app/lib/meadow/config/runtime.ex
+++ b/app/lib/meadow/config/runtime.ex
@@ -5,17 +5,7 @@ defmodule Meadow.Config.Runtime do
 
   alias Meadow.Config.Pipeline
 
-  @config_map %{
-    ezid: "infrastructure/ezid",
-    honeybadger: "infrastructure/honeybadger",
-    iiif: "infrastructure/iiif",
-    index: "infrastructure/index",
-    inference: "infrastructure/inference",
-    ldap: "infrastructure/ldap",
-    meadow: "config/meadow",
-    nusso: "infrastructure/nusso",
-    wildcard_ssl: "config/wildcard_ssl"
-  }
+  import Meadow.Config.Secrets
 
   # TODO: UPDATE ALL get_secret(:meadow, ["dc",...]) to use DC secrets
 
@@ -102,7 +92,7 @@ defmodule Meadow.Config.Runtime do
       pubsub_server: Meadow.PubSub
 
     config :meadow, Meadow.Search.Cluster,
-      url: get_secret(:meadow, ["index", "index_endpoint"]),
+      url: get_secret(:index, ["endpoint"]),
       default_options: [
         timeout: 20_000,
         recv_timeout: 90_000
@@ -302,77 +292,4 @@ defmodule Meadow.Config.Runtime do
 
     :ok
   end
-
-  def get_secret(config, path, default \\ nil) do
-    secrets =
-      case :ets.lookup(:secret_cache, config) |> Keyword.get(config) do
-        nil ->
-          loaded = load_config(@config_map[config])
-          :ets.insert(:secret_cache, {config, loaded})
-          loaded
-
-        value ->
-          value
-      end
-
-    case get_in(secrets, path) do
-      nil -> default
-      secret -> secret
-    end
-  end
-
-  defp load_config(config_path) do
-    System.get_env("SECRETS_PATH", nil) |> load_config(config_path)
-  end
-
-  defp load_config(nil, config_path), do: retrieve_config(config_path)
-
-  defp load_config(prefix, config_path),
-    do: Path.join(Enum.reject([prefix, config_path], &is_nil/1)) |> retrieve_config()
-
-  defp retrieve_config(path) do
-    case ExAws.SecretsManager.get_secret_value(path) |> ExAws.request() do
-      {:ok, %{"SecretString" => secret_string}} -> Jason.decode!(secret_string)
-      {:error, _} -> nil
-    end
-  end
-
-  def environment do
-    if function_exported?(Mix, :env, 0), do: Mix.env(), else: :prod
-  end
-
-  def prefix do
-    env =
-      cond do
-        System.get_env("RELEASE_NAME") -> nil
-        function_exported?(Mix, :env, 0) -> Mix.env()
-        true -> nil
-      end
-
-    [System.get_env("DEV_PREFIX"), env] |> Enum.reject(&is_nil/1) |> Enum.join("-")
-  end
-
-  def prefix(val), do: [prefix(), to_string(val)] |> reject_empty() |> Enum.join("-")
-  #  defp atom_prefix(val), do: prefix(val) |> String.to_atom()
-  defp reject_empty(list), do: Enum.reject(list, &(is_nil(&1) or &1 == ""))
-
-  defp environment_int(key, default) do
-    case System.get_env(key) do
-      nil -> default
-      val -> String.to_integer(val)
-    end
-  end
-
-  def project_root do
-    if Code.loaded?(Mix),
-      do: Path.dirname(Path.dirname(Mix.Project.build_path())),
-      else: Path.dirname(:code.priv_dir(:meadow))
-  end
-
-  defp priv_path(path) do
-    case :code.priv_dir(:meadow) do
-      {:error, :bad_name} -> Path.join([".", "priv", path])
-      priv_dir -> priv_dir |> to_string() |> Path.join(path)
-    end
-  end
 end
diff --git a/app/lib/meadow/config/runtime/dev.ex b/app/lib/meadow/config/runtime/dev.ex
index 0fc53129e..ea7e953e7 100644
--- a/app/lib/meadow/config/runtime/dev.ex
+++ b/app/lib/meadow/config/runtime/dev.ex
@@ -3,7 +3,7 @@ defmodule Meadow.Config.Runtime.Dev do
   Load and apply Meadow's runtime configuration for the dev environment
   """
 
-  import Meadow.Config.Runtime
+  import Meadow.Config.Secrets
 
   defp fetch_cert do
     cert_path = project_root() |> Path.join("priv/cert")
diff --git a/app/lib/meadow/config/runtime/test.ex b/app/lib/meadow/config/runtime/test.ex
index d154b1f4b..14bd76a80 100644
--- a/app/lib/meadow/config/runtime/test.ex
+++ b/app/lib/meadow/config/runtime/test.ex
@@ -14,12 +14,7 @@ defmodule Meadow.Config.Runtime.Test do
 
     config :meadow,
       index_interval: 1234,
-      mediaconvert_client: MediaConvert.Mock,
-      streaming_url: "https://test-streaming-url/",
-      iiif_server_url: "http://localhost:8184/iiif/3/",
-      iiif_manifest_url_deprecated:
-        "http://test-pyramids.s3.localhost.localstack.cloud:4566/public/",
-      digital_collections_url: "https://fen.rdc-staging.library.northwestern.edu/"
+      mediaconvert_client: MediaConvert.Mock
 
     # Configures lambda scripts
     config :meadow, :lambda,
@@ -38,15 +33,6 @@ defmodule Meadow.Config.Runtime.Test do
       required_checksum_tags: ["computed-md5"],
       checksum_wait_timeout: 15_000
 
-    config :meadow,
-      ark: %{
-        default_shoulder: "ark:/12345/nu2",
-        user: "mockuser",
-        password: "mockpassword",
-        target_url: "https://devbox.library.northwestern.edu:3333/items/",
-        url: "http://localhost:3944/"
-      }
-
     config :meadow, :elasticsearch_retry,
       interval: 100,
       max_retries: 3
@@ -54,11 +40,6 @@ defmodule Meadow.Config.Runtime.Test do
     config :authoritex, authorities: [Authoritex.Mock, NUL.Authority]
 
     config :meadow, Meadow.Repo,
-      username: "docker",
-      password: "d0ck3r",
-      database: "meadow",
-      hostname: "localhost",
-      port: 5432,
       show_sensitive_data_on_connection_error: true,
       timeout: 60_000,
       connect_timeout: 60_000,
@@ -77,26 +58,6 @@ defmodule Meadow.Config.Runtime.Test do
       ],
       iiif_distribution_id: nil
 
-    config :ueberauth, Ueberauth,
-      providers: [
-        nusso:
-          {Ueberauth.Strategy.NuSSO,
-           [
-             base_url: "https://northwestern-dev.apigee.net/agentless-websso/",
-             callback_path: "/auth/nusso/callback",
-             consumer_key: "test-sso-key",
-             include_attributes: false
-           ]}
-      ]
-
-    config :exldap, :settings,
-      base: "OU=test,DC=library,DC=northwestern,DC=edu",
-      server: "localhost",
-      port: 389,
-      user_dn: "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu",
-      password: "d0ck3rAdm1n!",
-      ssl: false
-
     config :ex_unit,
       assert_receive_timeout: 500
 
diff --git a/app/lib/meadow/config/secrets.ex b/app/lib/meadow/config/secrets.ex
new file mode 100644
index 000000000..9b3798495
--- /dev/null
+++ b/app/lib/meadow/config/secrets.ex
@@ -0,0 +1,91 @@
+defmodule Meadow.Config.Secrets do
+  @moduledoc """
+  Functions for retrieving and loading configuration from AWS Secrets Manager and
+  the local runtime environment
+  """
+
+  @config_map %{
+    ezid: "infrastructure/ezid",
+    honeybadger: "infrastructure/honeybadger",
+    iiif: "infrastructure/iiif",
+    index: "infrastructure/index",
+    inference: "infrastructure/inference",
+    ldap: "infrastructure/ldap",
+    meadow: "config/meadow",
+    nusso: "infrastructure/nusso",
+    wildcard_ssl: "config/wildcard_ssl"
+  }
+
+  def get_secret(config, path, default \\ nil) do
+    secrets =
+      case :ets.lookup(:secret_cache, config) |> Keyword.get(config) do
+        nil ->
+          loaded = load_config(@config_map[config])
+          :ets.insert(:secret_cache, {config, loaded})
+          loaded
+
+        value ->
+          value
+      end
+
+    case get_in(secrets, path) do
+      nil -> default
+      secret -> secret
+    end
+  end
+
+  defp load_config(config_path) do
+    System.get_env("SECRETS_PATH", nil) |> load_config(config_path)
+  end
+
+  defp load_config(nil, config_path), do: retrieve_config(config_path)
+
+  defp load_config(prefix, config_path),
+    do: Path.join(Enum.reject([prefix, config_path], &is_nil/1)) |> retrieve_config()
+
+  defp retrieve_config(path) do
+    case ExAws.SecretsManager.get_secret_value(path) |> ExAws.request() do
+      {:ok, %{"SecretString" => secret_string}} -> Jason.decode!(secret_string)
+      {:error, _} -> nil
+    end
+  end
+
+  def environment do
+    if function_exported?(Mix, :env, 0), do: Mix.env(), else: :prod
+  end
+
+  def prefix do
+    env =
+      cond do
+        System.get_env("RELEASE_NAME") -> nil
+        function_exported?(Mix, :env, 0) -> Mix.env()
+        true -> nil
+      end
+
+    [System.get_env("DEV_PREFIX"), env] |> Enum.reject(&is_nil/1) |> Enum.join("-")
+  end
+
+  def prefix(val), do: [prefix(), to_string(val)] |> reject_empty() |> Enum.join("-")
+  #  defp atom_prefix(val), do: prefix(val) |> String.to_atom()
+  defp reject_empty(list), do: Enum.reject(list, &(is_nil(&1) or &1 == ""))
+
+  def environment_int(key, default) do
+    case System.get_env(key) do
+      nil -> default
+      val -> String.to_integer(val)
+    end
+  end
+
+  def project_root do
+    if Code.loaded?(Mix),
+      do: Path.dirname(Path.dirname(Mix.Project.build_path())),
+      else: Path.dirname(:code.priv_dir(:meadow))
+  end
+
+  def priv_path(path) do
+    case :code.priv_dir(:meadow) do
+      {:error, :bad_name} -> Path.join([".", "priv", path])
+      priv_dir -> priv_dir |> to_string() |> Path.join(path)
+    end
+  end
+end
diff --git a/app/lib/meadow/utils/aws.ex b/app/lib/meadow/utils/aws.ex
index 754ad9227..0ca267589 100644
--- a/app/lib/meadow/utils/aws.ex
+++ b/app/lib/meadow/utils/aws.ex
@@ -5,7 +5,7 @@ defmodule Meadow.Utils.AWS do
   Utility functions for AWS requests and object management
   """
   alias Meadow.Config
-  alias Meadow.Config.Runtime
+  alias Meadow.Config.Secrets
   alias Meadow.Error
   alias Meadow.Utils.AWS.MultipartCopy
   alias Meadow.Utils.Pairtree
@@ -282,5 +282,5 @@ defmodule Meadow.Utils.AWS do
     end
   end
 
-  defp prefix, do: Runtime.prefix()
+  defp prefix, do: Secrets.prefix()
 end
diff --git a/infrastructure/localstack/environment_config.tf b/infrastructure/localstack/environment_config.tf
index e28d3d60a..538fa43e3 100644
--- a/infrastructure/localstack/environment_config.tf
+++ b/infrastructure/localstack/environment_config.tf
@@ -1,51 +1,112 @@
 locals {
-  project       = "meadow"
-  port_offset   = 0 # terraform.workspace == "test" ? 2 : 1
-
-  computed_secrets = {
-    db   = {
-      host        = "localhost"
-      port        = 5432 + local.port_offset
-      user        = "docker"
-      password    = "d0ck3r"
+  secrets = {
+    "config/meadow" = {
+      buckets = {
+        ingest                = "test-ingest"
+        preservation          = "test-preservation"
+        preservation_check    = "test-preservation-checks"
+        pyramid               = "test-pyramids"
+        sitemap               = "test-sitemaps"
+        streaming             = "test-streaming"
+        upload                = "test-upload"
+      }
+
+      db = {
+        database    = "postgres"
+        host        = "localhost"
+        port        = 5432
+        user        = "docker"
+        password    = "d0ck3r"
+      }
+
+      dc = {
+        base_url = "https://dc.dev.library.northwestern.edu/"
+      }
+
+      dc_api = {
+        v2 = {
+          api_token_secret    = "TEST_SECRET"
+          api_token_ttl       = 300
+          base_url            = "http://dcapi-test.northwestern.edu"
+        }
+        iiif_distribution_id = null
+      }
+
+      ezid = {
+        password          = "apitest"
+        shoulder          = "ark:/99999/fk4"
+        target_base_url   = "https://dc.dev.library.northwestern.edu/items/"
+        url               = "https://ezid.cdlib.org/"
+        user              = "apitest"
+      }
+
+      geonames = {
+        username = "nul_rdc"
+      }
+
+      iiif = {
+        base_url          = "http://localhost:8184/iiif/3/"
+        distribution_id   = null
+        manifest_url      = "http://test-pyramids.s3.localhost.localstack.cloud:4566/public/"
+      }
+
+      mediaconvert = {
+        queue       = "arn:aws:mediaconvert:us-east-1:000000000000:queues/meadow"
+        role_arn    = "arn:aws:iam::000000000000:role/meadow-transcode-role"
+      }
+
+      streaming = {
+        base_url          = "https://test-streaming-url/"
+        distribution_id   = "Z7Q9N4L3X8P5J2"
+      }
+
+      work_archiver = {
+        endpoint = null
+      }
+
+    }
+
+    "infrastructure/ezid" = {
+      password    = "apitest"
+      shoulder    = "ark:/99999/fk4"
+      user        = "apitest"
     }
 
-    index = {
-      index_endpoint    = "http://localhost:${9200 + local.port_offset}"
-      kibana_endpoint   = "http://localhost:${5601 + local.port_offset}"
+    "infrastructure/index" = {
+      endpoint = "http://localhost:9200"
     }
-    ldap = {
-      host       = "localhost"
-      base       = "DC=library,DC=northwestern,DC=edu"
-      port       = 389 + local.port_offset
-      user_dn    = "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu"
-      password   = "d0ck3rAdm1n!"
-      ssl        = "false"
+
+    # "infrastructure/inference" = {
+    #   endpoints = {
+    #     endpoint = "https://bedrock-runtime.us-east-1.amazonaws.com/model/cohere.embed-multilingual-v3/invoke"
+    #     name = "cohere.embed-multilingual-v3"
+    #   }
+
+    # }
+
+    "infrastructure/ldap" = {
+        base        = "OU=test,DC=library,DC=northwestern,DC=edu"
+        server      = "localhost"
+        port        = 389
+        user_dn     = "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu"
+        password    = "d0ck3rAdm1n!"
+        ssl         = false
     }
-  }
 
-  config_secrets = merge(var.config_secrets, local.computed_secrets)
+    "infrastructure/nusso" = {
+      api_key   = "test-sso-key"
+      base_url  = "https://northwestern-dev.apigee.net/agentless-websso/"
+    }
+  }
 }
 
 resource "aws_secretsmanager_secret" "config_secrets" {
-  name    = "config/meadow"
-  description = "Meadow configuration secrets"
-}
-
-resource "aws_secretsmanager_secret" "ssl_certificate" {
-  name = "config/wildcard_ssl"
-  description = "Wildcard SSL certificate and private key"
+  for_each = local.secrets
+  name     = each.key
 }
 
 resource "aws_secretsmanager_secret_version" "config_secrets" {
-  secret_id = aws_secretsmanager_secret.config_secrets.id
-  secret_string = jsonencode(local.config_secrets)
-}
-
-resource "aws_secretsmanager_secret_version" "ssl_certificate" {
-  secret_id       = aws_secretsmanager_secret.ssl_certificate.id
-  secret_string   = jsonencode({
-    certificate = file(var.ssl_certificate_file)
-    key         = file(var.ssl_key_file)
-  })
+  for_each = local.secrets
+  secret_id = aws_secretsmanager_secret.config_secrets[each.key].id
+  secret_string = jsonencode(each.value)
 }
diff --git a/infrastructure/localstack/test.tfvars b/infrastructure/localstack/test.tfvars
index 0e7a7c1b2..886898c51 100644
--- a/infrastructure/localstack/test.tfvars
+++ b/infrastructure/localstack/test.tfvars
@@ -1,27 +1,2 @@
-config_secrets = {
-  ezid = {
-    password    = "apitest"
-    shoulder    = "ark:/99999/fk4"
-    user        = "apitest"
-  }
-
-  geonames = {
-    username    = ""
-  }
-
-  iiif = {
-    manifest_url = "http://test-pyramids.s3.localhost.localstack.cloud:4568/public/"
-  }
-
-  nusso = {
-    api_key     = "test-sso-key"
-    base_url    = "https://northwestern-dev.apigee.net/agentless-websso/"
-  }
-
-  streaming = {
-    base_url    = "https://test-streaming-url/"
-  }
-}
-
 ssl_certificate_file    = "/dev/null"
 ssl_key_file            = "/dev/null"
diff --git a/infrastructure/localstack/variables.tf b/infrastructure/localstack/variables.tf
index df51a63be..3ab5dd271 100644
--- a/infrastructure/localstack/variables.tf
+++ b/infrastructure/localstack/variables.tf
@@ -1,8 +1,3 @@
-variable "config_secrets" {
-  type    = map(map(string))
-  default = {}
-}
-
 variable "ssl_certificate_file" {
   type    = string
   default = "../../miscellany/devbox_cert/dev.rdc.wildcard.full.pem"