Merge pull request #4093 from nulib/5160-truncate

Truncate embedding text
nulib · Aug 14, 2024 · 8f3f145 · 8f3f145
2 parents 5ab2c1c + 6a15e58
commit 8f3f145
Showing 1 changed file with 13 additions and 13 deletions.
diff --git a/app/lib/meadow/indexing/v2/work.ex b/app/lib/meadow/indexing/v2/work.ex
@@ -78,28 +78,26 @@ defmodule Meadow.Indexing.V2.Work do
   end
 
   @embedding_keys [
-    :abstract,
+    :title,
+    :description,
+    :collection,
     :alternate_title,
     :caption,
-    :collection,
+    :table_of_contents,
+    :abstract,
     :contributor,
     :creator,
     :date_created,
-    :description,
     :genre,
-    # :keywords,
+    :subject,
+    :style_period,
     :language,
     :location,
-    :physical_description_material,
-    :physical_description_size,
     :publisher,
     :scope_and_contents,
-    :source,
-    :subject,
-    :style_period,
-    :table_of_contents,
-    :title,
-    :technique
+    :technique,
+    :physical_description_material,
+    :physical_description_size,
   ]
 
   defp prepare_embedding_field(map) do
@@ -115,10 +113,12 @@ defmodule Meadow.Indexing.V2.Work do
       end)
       |> Enum.join("\n")
 
-    Map.put(map, :embedding_text, value)
+    Map.put(map, :embedding_text_length, String.length(value))
+    |> Map.put(:embedding_text, String.slice(value, 0, 2048))
   end
 
   defp prepare_embedding_value(%{label: v}), do: prepare_embedding_value(v)
+  defp prepare_embedding_value(%{title: v}), do: prepare_embedding_value(v)
 
   defp prepare_embedding_value([]), do: []