fix(ai): Do not store generated embeddings in the document store

postalsys · Sep 20, 2023 · 9638480 · 9638480
1 parent a787cbf
commit 9638480
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 5 deletions.
diff --git a/package.json b/package.json
@@ -50,7 +50,7 @@
         "@hapi/vision": "7.0.3",
         "@phc/pbkdf2": "1.1.14",
         "@postalsys/certs": "1.0.5",
-        "@postalsys/email-ai-tools": "1.3.2",
+        "@postalsys/email-ai-tools": "1.3.4",
         "@postalsys/email-text-tools": "2.1.1",
         "@postalsys/hecks": "3.0.0-fork.3",
         "@postalsys/templates": "1.0.5",
@@ -86,7 +86,7 @@
         "license-checker": "25.0.1",
         "mailparser": "3.6.5",
         "mailsplit": "5.4.0",
-        "marked": "9.0.2",
+        "marked": "9.0.3",
         "minimist": "1.2.8",
         "msgpack5": "6.0.2",
         "murmurhash": "2.0.1",

diff --git a/server.js b/server.js
@@ -1424,7 +1424,20 @@ async function onCommand(worker, message) {
                 throw new Error(`OpenAI API key is not set`);
             }
 
-            return { chunks: await generateEmbeddings(message.data.message, openAiAPIKey, requestOpts) };
+            const embeddings = await generateEmbeddings(message.data.message, openAiAPIKey, requestOpts);
+            if (!Array.isArray(embeddings?.embeddings)) {
+                return false;
+            }
+
+            for (let value of embeddings.embeddings) {
+                for (const key of Object.keys(value)) {
+                    if (/^_/.test(key)) {
+                        delete value[key];
+                    }
+                }
+            }
+
+            return embeddings;
         }
 
         case 'openAiDefaultPrompt': {

diff --git a/views/accounts/register/imap.hbs b/views/accounts/register/imap.hbs
@@ -10,7 +10,7 @@
                 <div class="form-group">
                     <label for="name">{{_ "Your name" }}</label>
                     <input type="text" class="form-control {{#if errors.name}}is-invalid{{/if}}" id="name" name="name"
-                        value="{{values.name}}" autofocus />
+                        value="{{values.name}}" {{#unless values.name}}autofocus{{/unless}} />
                     {{#if errors.name}}
                     <span class="invalid-feedback">{{errors.name}}</span>
                     {{/if}}
@@ -20,7 +20,7 @@
                 <div class="form-group">
                     <label for="email">{{_ "Email address" }}</label>
                     <input type="email" class="form-control {{#if errors.email}}is-invalid{{/if}}" id="email"
-                        name="email" value="{{values.email}}" required />
+                        name="email" value="{{values.email}}" {{#if values.name}}autofocus{{/if}} required />
                     {{#if errors.email}}
                     <span class="invalid-feedback">{{errors.email}}</span>
                     {{/if}}

diff --git a/workers/documents.js b/workers/documents.js
@@ -319,6 +319,9 @@ const documentsWorker = new Worker(
                     }
                 }
 
+                // Skip embeddings if set for document store (nested dense cosine vectors can not be indexed, must be separate documents)
+                delete messageData.embeddings;
+
                 let emailDocument = await preProcess.run(messageData);
                 if (!emailDocument) {
                     // skip