Move all dynamic secrets to Meadow.Config.Runtime namespace

Remove janky `Env` module and `Hush` dependency

Remove Hush from test_helper.exs

Consolidate component configurations into compile-time/runtime

Push more test configs into localstack secrets manager

Turn off warnings on lambda logging

.github/workflows/test.yml

@@ -126,7 +126,7 @@ jobs:
           POSTGRES_USER: docker
           POSTGRES_PASSWORD: d0ck3r
         options: >-
-          --health-cmd "pg_isready -U postgres"
+          --health-cmd "pg_isready -U docker"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
@@ -221,6 +221,8 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-npm-v6-${{ hashFiles('**/package-lock.json') }}
             ${{ runner.os }}-npm-v6-
+      - name: Set SECRET_KEY_BASE
+        run: echo "SECRET_KEY_BASE=$(openssl rand -hex 32)" >> $GITHUB_ENV
       - name: Elixir Static Analysis
         run: mix credo
         working-directory: app

.gitignore

@@ -49,6 +49,7 @@ npm-debug.log
 *.tfvars
 
 /infrastructure/**/.terraform/
+/infrastructure/**/*.tfstate*
 /infrastructure/**/_build/
 /infrastructure/**/build/
 /infrastructure/**/builds/

app/config/.credo.exs

@@ -7,6 +7,7 @@
         excluded: []
       },
       checks: [
+        {Credo.Check.Design.TagTODO, false},
         {Credo.Check.Refactor.MapInto, false},
         {Credo.Check.Warning.LazyLogging, false}
       ]

app/config/config.exs

@@ -1,207 +1,44 @@
-# This file is responsible for configuring your application
-# and its dependencies with the aid of the Mix.Config module.
+# Meadow configuration takes place across multiple files.
 #
-# This configuration file is loaded before any dependency and
-# is restricted to this project.
+# Compile-time configs are evaluated when Meadow is compiled, with
+# higher priority given to files farther down the list (i.e., environment-
+# specific configs take precedence over global config).
+#
+#   config/config.exs
+#   config/[environment].exs
+#
+# Runtime configs work the same way, except they are set at runtime.
+# This means they can read configuration data that may not be available
+# at compile-time, or that may change. This includes files on disk,
+# the runtime system environment, or remote datastores such as AWS
+# Secrets Manager.
+#
+#   lib/meadow/config/runtime.ex
+#   lib/meadow/config/runtime/[environment].ex
+#   config/[environment].local.exs
+#
+# When one configuration takes precedence over other, keyword lists are
+# deep merged, while other values (e.g., maps) are overwritten.
+#
+# It is _strongly_ suggested that component configurations be done entirely
+# at compile-time _or_ runtime and not some of each, e.g.:
+# don't try to
+# ```
+# config :meadow, :component, [static properties]
+# ```
+# at compile time and
+# ```
+# config :meadow, :component, [dynamic properties]
+# ```
+# at runtime.
 
 # General application configuration
 import Config
-import Env
 
-alias Meadow.Utils.Hush.Transformer, as: CustomTransformer
+alias Meadow.Pipeline.Actions
 
 config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase
 
-config :meadow,
-  ecto_repos: [Meadow.Repo],
-  environment: Mix.env(),
-  environment_prefix: prefix()
-
-config :meadow, Meadow.Repo,
-  migration_primary_key: [
-    name: :id,
-    type: :binary_id,
-    autogenerate: false,
-    read_after_writes: true,
-    default: {:fragment, "uuid_generate_v4()"}
-  ],
-  migration_timestamps: [type: :utc_datetime_usec],
-  username: aws_secret("meadow", dig: ["db", "user"], default: "docker"),
-  password: aws_secret("meadow", dig: ["db", "password"], default: "d0ck3r"),
-  database: aws_secret("meadow", dig: ["db", "database"], default: prefix("meadow")),
-  hostname: aws_secret("meadow", dig: ["db", "host"], default: "localhost"),
-  port: aws_secret("meadow", dig: ["db", "port"], default: 5432)
-
-# Configures the endpoint
-config :meadow, MeadowWeb.Endpoint,
-  url: [host: "localhost"],
-  secret_key_base: "C7BC/yBsTCe/PaJ9g0krwlQrNZZV2r3jSjeuGCeIu9mfNE+4bPcNPHiINQtIQk/B",
-  render_errors: [view: MeadowWeb.ErrorView, accepts: ~w(html json)],
-  pubsub_server: Meadow.PubSub,
-  live_view: [signing_salt: "C7BC/yBsTCe/PaJ9g0krwlQrNZZV2r3jSjeuGCeIu9mfNE+4bPcNPHiINQtIQk/B"]
-
-# Configures the Search cluster
-config :meadow, Meadow.Search.Cluster,
-  url:
-    aws_secret("meadow",
-      dig: ["search", "cluster_endpoint"],
-      default: "http://localhost:9201"
-    ),
-  indexes: [
-    %{
-      name: prefix("dc-v2-work"),
-      settings: "priv/search/v2/settings/work.json",
-      version: 2,
-      schemas: [Meadow.Data.Schemas.Work],
-      pipeline: prefix("dc-v2-work-pipeline")
-    },
-    %{
-      name: prefix("dc-v2-file-set"),
-      settings: "priv/search/v2/settings/file_set.json",
-      version: 2,
-      schemas: [Meadow.Data.Schemas.FileSet]
-    },
-    %{
-      name: prefix("dc-v2-collection"),
-      settings: "priv/search/v2/settings/collection.json",
-      version: 2,
-      schemas: [Meadow.Data.Schemas.Collection]
-    }
-  ],
-  default_options: [
-    timeout: 20_000,
-    recv_timeout: 30_000
-  ],
-  bulk_page_size: 200,
-  bulk_wait_interval: 500,
-  json_encoder: Ecto.Jason,
-  embedding_model_id:
-    aws_secret("meadow",
-      dig: ["search", "embedding_model_id"],
-      default: nil
-    ),
-  embedding_dimensions:
-    aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil),
-  embedding_text_fields: [
-    :title,
-    :alternate_title,
-    :description,
-    :collection,
-    :creator,
-    :contributor,
-    :date_created,
-    :genre,
-    :subject,
-    :style_period,
-    :language,
-    :location,
-    :publisher,
-    :technique,
-    :physical_description_material,
-    :physical_description_size,
-    :caption,
-    :table_of_contents,
-    :scope_and_contents,
-    :abstract,
-  ]
-
-config :meadow,
-  ark: %{
-    default_shoulder: aws_secret("meadow", dig: ["ezid", "shoulder"], default: "ark:/12345/nu1"),
-    user: aws_secret("meadow", dig: ["ezid", "user"], default: "ark_user"),
-    password: aws_secret("meadow", dig: ["ezid", "password"], default: "ark_password"),
-    target_url:
-      aws_secret("meadow",
-        dig: ["ezid", "target_base_url"],
-        default: "https://devbox.library.northwestern.edu:3333/items/"
-      ),
-    url: aws_secret("meadow", dig: ["ezid", "url"], default: "http://localhost:3943")
-  },
-  ingest_bucket: prefix("ingest"),
-  upload_bucket: prefix("uploads"),
-  pipeline_delay: :timer.seconds(5),
-  preservation_bucket: prefix("preservation"),
-  preservation_check_bucket: prefix("preservation-checks"),
-  pyramid_bucket: prefix("pyramids"),
-  streaming_bucket: prefix("streaming"),
-  streaming_url:
-    aws_secret("meadow",
-      dig: ["streaming", "base_url"],
-      default: "https://#{prefix()}-streaming.s3.amazonaws.com/"
-    ),
-  mediaconvert_client: MediaConvert,
-  multipart_upload_concurrency: System.get_env("MULTIPART_UPLOAD_CONCURRENCY", "10"),
-  iiif_server_url:
-    aws_secret("meadow",
-      dig: ["iiif", "base_url"],
-      default: "https://iiif.dev.rdc.library.northwestern.edu/iiif/3/#{prefix()}"
-    ),
-  iiif_manifest_url_deprecated:
-    aws_secret("meadow",
-      dig: ["iiif", "manifest_url"],
-      default: "https://#{prefix()}-pyramids.s3.amazonaws.com/public/"
-    ),
-  iiif_distribution_id: aws_secret("meadow", dig: ["iiif", "distribution_id"], default: nil),
-  digital_collections_url:
-    aws_secret("meadow",
-      dig: ["dc", "base_url"],
-      default: "https://dc.rdc-staging.library.northwestern.edu/"
-    ),
-  progress_ping_interval: System.get_env("PROGRESS_PING_INTERVAL", "1000"),
-  validation_ping_interval: System.get_env("VALIDATION_PING_INTERVAL", "1000"),
-  shared_links_index: prefix("shared_links"),
-  pyramid_tiff_working_dir: System.tmp_dir!(),
-  streaming_distribution_id:
-    aws_secret("meadow", dig: ["streaming", "distribution_id"], default: nil),
-  work_archiver_endpoint: aws_secret("meadow", dig: ["work_archiver", "endpoint"], default: "")
-
-config :exldap, :settings,
-  server: aws_secret("meadow", dig: ["ldap", "host"], default: "localhost"),
-  base:
-    aws_secret("meadow",
-      dig: ["ldap", "base"],
-      default: "DC=library,DC=northwestern,DC=edu"
-    ),
-  port: aws_secret("meadow", dig: ["ldap", "port"], cast: :integer, default: 390),
-  user_dn:
-    aws_secret("meadow",
-      dig: ["ldap", "user_dn"],
-      default: "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu"
-    ),
-  password: aws_secret("meadow", dig: ["ldap", "password"], default: "d0ck3rAdm1n!"),
-  ssl: aws_secret("meadow", dig: ["ldap", "ssl"], cast: :boolean, default: false)
-
-config :meadow,
-  transcoding_presets: %{
-    audio: [
-      %{NameModifier: "-high", Preset: "meadow-audio-high"},
-      %{NameModifier: "-medium", Preset: "meadow-audio-medium"}
-    ],
-    video: [
-      %{NameModifier: "-1080", Preset: "meadow-video-high"},
-      %{NameModifier: "-720", Preset: "meadow-video-medium"},
-      %{NameModifier: "-540", Preset: "meadow-video-low"}
-    ]
-  }
-
-# Configure checksum requirements
-config :meadow,
-  required_checksum_tags: ["computed-md5"],
-  checksum_wait_timeout: 3_600_000
-
-# Configure Lambda-based actions
-lambda_from_ssm = fn lambda, function ->
-  {:lambda, aws_secret("meadow", dig: ["pipeline", lambda], default: "#{function}:$LATEST")}
-end
-
-config :meadow, :lambda,
-  digester: lambda_from_ssm.("digester", "digester"),
-  exif: lambda_from_ssm.("exif", "exif"),
-  frame_extractor: lambda_from_ssm.("frame_extractor", "frame-extractor"),
-  mediainfo: lambda_from_ssm.("mediainfo", "mediainfo"),
-  mime_type: lambda_from_ssm.("mime_type", "mime-type"),
-  tiff: lambda_from_ssm.("tiff", "pyramid-tiff")
-
 # Configures Elixir's Logger
 config :logger, :console,
   format: "$time $metadata[$level] $message\n",
@@ -210,8 +47,6 @@ config :logger, :console,
 # Use Jason for JSON parsing in Phoenix
 config :phoenix, :json_library, Jason
 
-config :ueberauth, Ueberauth, providers: [nusso: {Ueberauth.Strategy.NuSSO, []}]
-
 config :authoritex,
   authorities: [
     Authoritex.FAST.CorporateName,
@@ -235,20 +70,6 @@ config :authoritex,
     NUL.Authority
   ]
 
-config :honeybadger,
-  api_key: environment_secret("HONEYBADGER_API_KEY", default: "DO_NOT_REPORT"),
-  environment_name: System.get_env("HONEYBADGER_ENVIRONMENT", to_string(Mix.env())),
-  revision: System.get_env("HONEYBADGER_REVISION", ""),
-  repos: [Meadow.Repo],
-  breadcrumbs_enabled: true,
-  filter: Meadow.Error.Filter,
-  exclude_envs: [:dev, :test]
-
-config :ex_aws,
-  access_key_id: [:instance_role],
-  secret_access_key: [:instance_role],
-  region: System.get_env("AWS_REGION", "us-east-1")
-
 config :httpoison_retry, wait: 50
 
 config :meadow, :extra_mime_types, %{
@@ -266,28 +87,26 @@ config :meadow, :extra_mime_types, %{
   "xml" => "application/xml"
 }
 
-config :meadow,
-  dc_api: [
-    v2: aws_secret("meadow", dig: ["dc_api", "v2"])
+config :meadow, Meadow.Pipeline,
+  actions: [
+    Actions.IngestFileSet,
+    Actions.ExtractMimeType,
+    Actions.InitializeDispatch,
+    Actions.GenerateFileSetDigests,
+    Actions.ExtractExifMetadata,
+    Actions.CopyFileToPreservation,
+    Actions.CreateDerivativeCopy,
+    Actions.CreatePyramidTiff,
+    Actions.ExtractMediaMetadata,
+    Actions.CreateTranscodeJob,
+    Actions.TranscodeComplete,
+    Actions.GeneratePosterImage,
+    Actions.FileSetComplete
   ]
 
-config :hush,
-  transformers_override: true,
-  transformers: [
-    CustomTransformer.Dig,
-    CustomTransformer.Default,
-    CustomTransformer.Split,
-    Hush.Transformer.Apply,
-    CustomTransformer.Cast,
-    Hush.Transformer.ToFile
+config :ueberauth, Ueberauth,
+  providers: [
+    nusso: {Ueberauth.Strategy.NuSSO, []}
   ]
 
-import_config "pipeline.exs"
-
-# Import environment specific config. This must remain at the bottom
-# of this file so it overrides the configuration defined above.
-
 import_config("#{Mix.env()}.exs")
-
-if File.exists?("config/#{Mix.env()}.local.exs"),
-  do: import_config("#{Mix.env()}.local.exs")