Merge pull request #4103 from nulib/adjust-embedding-text

Make embedding keys configurable
nulib · Aug 15, 2024 · 692f006 · 692f006
2 parents ab1cdaf + df226dd
commit 692f006
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 25 deletions.
diff --git a/app/config/config.exs b/app/config/config.exs
@@ -81,7 +81,29 @@ config :meadow, Meadow.Search.Cluster,
       default: nil
     ),
   embedding_dimensions:
-    aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil)
+    aws_secret("meadow", dig: ["search", "embedding_dimensions"], default: nil),
+  embedding_text_fields: [
+    :title,
+    :description,
+    :collection,
+    # :alternate_title,
+    # :caption,
+    # :table_of_contents,
+    # :abstract,
+    # :contributor,
+    # :creator,
+    # :date_created,
+    # :genre,
+    # :subject,
+    # :style_period,
+    # :language,
+    # :location,
+    # :publisher,
+    # :scope_and_contents,
+    # :technique,
+    # :physical_description_material,
+    # :physical_description_size,
+  ]
 
 config :meadow,
   ark: %{

diff --git a/app/lib/meadow/indexing/v2/work.ex b/app/lib/meadow/indexing/v2/work.ex
@@ -5,6 +5,7 @@ defmodule Meadow.Indexing.V2.Work do
 
   alias Meadow.Data.FileSets
   alias Meadow.Data.Schemas.{ControlledMetadataEntry, NoteEntry, RelatedURLEntry}
+  alias Meadow.Search.Config
 
   def encode(work) do
     %{
@@ -77,32 +78,9 @@ defmodule Meadow.Indexing.V2.Work do
     |> prepare_embedding_field()
   end
 
-  @embedding_keys [
-    :title,
-    :description,
-    :collection,
-    :alternate_title,
-    :caption,
-    :table_of_contents,
-    :abstract,
-    :contributor,
-    :creator,
-    :date_created,
-    :genre,
-    :subject,
-    :style_period,
-    :language,
-    :location,
-    :publisher,
-    :scope_and_contents,
-    :technique,
-    :physical_description_material,
-    :physical_description_size,
-  ]
-
   defp prepare_embedding_field(map) do
     value =
-      @embedding_keys
+      Config.embedding_text_fields()
       |> Enum.reduce([], fn field_name, acc ->
         v = prepare_embedding_value(Map.get(map, field_name))
         [v | acc]

diff --git a/app/lib/meadow/search/config.ex b/app/lib/meadow/search/config.ex
@@ -76,6 +76,11 @@ defmodule Meadow.Search.Config do
     |> Keyword.get(:embedding_dimensions)
   end
 
+  def embedding_text_fields do
+    Application.get_env(:meadow, Meadow.Search.Cluster)
+    |> Keyword.get(:embedding_text_fields)
+  end
+
   def index_versions do
     index_configs()
     |> Enum.map(& &1.version)