Implement Semantic Search

app.arc

@@ -156,6 +156,8 @@ availabilityZoneCount 3
 volumeSize 10
 dedicatedMasterCount 3
 dedicatedMasterType t3.small.search
+# Use OpenSearch in sandbox mode; default is Elasticsearch.
+sandboxEngine opensearch
 
 @plugins
 plugin-remix

app/routes/_gcn.circulars._archive._index/route.tsx

@@ -38,12 +38,14 @@ import CircularsHeader from './CircularsHeader'
 import CircularsIndex from './CircularsIndex'
 import { DateSelector } from './DateSelectorMenu'
 import Hint from '~/components/Hint'
+import { feature } from '~/lib/env.server'
 import { getFormDataString } from '~/lib/utils'
 import { useFeature } from '~/root'
 
 import searchImg from 'nasawds/src/img/usa-icons-bg/search--white.svg'
 
 export async function loader({ request: { url } }: LoaderFunctionArgs) {
+  const useNLP = feature('CIRCULARS_USE_NLP')
   const { searchParams } = new URL(url)
   const query = searchParams.get('query') || undefined
   if (query) {
@@ -59,6 +61,7 @@ export async function loader({ request: { url } }: LoaderFunctionArgs) {
     limit,
     startDate,
     endDate,
+    useNLP,
   })
 
   return { page, ...results }

app/routes/_gcn.circulars/circulars.server.ts

@@ -127,41 +127,57 @@ export async function search({
   limit,
   startDate,
   endDate,
+  useNLP,
 }: {
   query?: string
   page?: number
   limit?: number
   startDate?: string
   endDate?: string
+  useNLP?: boolean
 }): Promise<{
   items: CircularMetadata[]
   totalPages: number
   totalItems: number
 }> {
   const client = await getSearch()
-
   const [startTime, endTime] = getValidDates(startDate, endDate)
 
-  const {
-    body: {
-      hits: {
-        total: { value: totalItems },
-        hits,
-      },
-    },
-  } = await client.search({
-    index: 'circulars',
-    body: {
-      query: {
+  const get_model_id_request = {
+    path: '/_ingest/pipeline',
+  }
+
+  let model_id = ''
+  try {
+    const resp = await client.http.get(get_model_id_request)
+
+    if (resp && resp.statusCode == 200) {
+      model_id =
+        resp.body['nlp-ingest-pipeline'].processors[0].text_embedding.model_id
+    } else {
+      console.log(
+        'Error. Could not deploy model. Returned with response: ',
+        resp
+      )
+    }
+  } catch (e) {
+    console.log('Error: ', e)
+  }
+
+  const nlpSearchQuery = query
+    ? {
         bool: {
-          must: query
-            ? {
-                multi_match: {
-                  query,
-                  fields: ['submitter', 'subject', 'body'],
+          must: [
+            {
+              neural: {
+                circular_embedding: {
+                  query_text: query,
+                  model_id,
+                  k: 100,
                 },
-              }
-            : undefined,
+              },
+            },
+          ],
           filter: {
             range: {
               createdOn: {
@@ -171,7 +187,42 @@ export async function search({
             },
           },
         },
+      }
+    : { match_all: {} }
+
+  const searchQuery = {
+    bool: {
+      must: query
+        ? {
+            multi_match: {
+              query,
+              fields: ['submitter', 'subject', 'body'],
+            },
+          }
+        : undefined,
+      filter: {
+        range: {
+          createdOn: {
+            gte: startTime,
+            lte: endTime,
+          },
+        },
+      },
+    },
+  }
+
+  const chosenQuery = useNLP ? nlpSearchQuery : searchQuery
+  const {
+    body: {
+      hits: {
+        total: { value: totalItems },
+        hits,
       },
+    },
+  } = await client.search({
+    index: 'circulars',
+    body: {
+      query: chosenQuery,
       fields: ['subject'],
       _source: false,
       sort: {