Skip to content

Commit

Permalink
Added embeddings checkbox to document store config page
Browse files Browse the repository at this point in the history
  • Loading branch information
@andris9
andris9 committed Sep 23, 2023
1 parent 18038b5 commit 0c40bd9
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 17 deletions.
2 changes: 1 addition & 1 deletion lib/account.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1741,7 +1741,7 @@ class Account {
}
};

for (let indexName of [index, `${index}.threads`]) {
for (let indexName of [index, `${index}.threads`, `${index}.embeddings`]) {
try {
deleteResult[indexName] = await client.deleteByQuery({
index: indexName,
Expand Down
6 changes: 3 additions & 3 deletions views/config/ai.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,15 @@
id="openAiGenerateEmbeddings" name="openAiGenerateEmbeddings" {{#if
values.openAiGenerateEmbeddings}}checked{{/if}} />
<label class="form-check-label" for="openAiGenerateEmbeddings">Generate vector embeddings for
the
email using OpenAI API.</label>
the email using OpenAI API.</label>
{{#if errors.openAiGenerateEmbeddings}}
<span class="invalid-feedback">{{errors.openAiGenerateEmbeddings}}</span>
{{/if}}
<small class="form-text text-muted">EmailEngine uses OpenAI API to generate text embeddings for
new emails. To decrease vector size, EmailEngine splits email text into chunks of up to 600
tokens and generates embeddings for each chunk separately. Embeddings are only available in
<code>messageNew</code> webhooks, and these are not stored in the Document Store.</small>
<code>messageNew</code> webhooks, and these are not stored in the Document Store by
default.</small>
</div>

<div class="text-right">
Expand Down
73 changes: 61 additions & 12 deletions views/config/document-store/index.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,21 +49,70 @@
</div>
</div>
</div>
<div class="card-body">

<div class="form-group form-check">
<div class="text-muted float-right code-link">[<a href="/admin/iframe/docs#/settings/postV1Settings"
target="_blank">documentStoreEnabled</a>]
<ul class="list-group list-group-flush mb-3">
<li class="list-group-item d-flex justify-content-between align-items-top">

<div class="form-group form-check mb-0">
<input type="checkbox" class="form-check-input {{#if errors.documentStoreEnabled}}is-invalid{{/if}}"
id="documentStoreEnabled" name="documentStoreEnabled" {{#if
values.documentStoreEnabled}}checked{{/if}} />
<label class="form-check-label" for="documentStoreEnabled">Enable syncing to the Document
Store.</label>
{{#if errors.documentStoreEnabled}}
<span class="invalid-feedback">{{errors.documentStoreEnabled}}</span>
{{/if}}
<small class="form-text text-muted">If enabled, then EmailEngine stores structured email content to
ElasticSearch. Stored data does include attachment metadata but not attachment contents.
<i>Enabling this option does not apply to already processed emails.</i></small>
</div>

<input type="checkbox" class="form-check-input {{#if errors.documentStoreEnabled}}is-invalid{{/if}}"
id="documentStoreEnabled" name="documentStoreEnabled" {{#if
values.documentStoreEnabled}}checked{{/if}} />
<label class="form-check-label" for="documentStoreEnabled">Enable syncing to the Document Store</label>
{{#if errors.documentStoreEnabled}}
<span class="invalid-feedback">{{errors.documentStoreEnabled}}</span>
{{/if}}
</div>
<div class="text-right">
<div class="text-muted code-link">[<a href="/admin/iframe/docs#/settings/postV1Settings"
target="_blank">documentStoreEnabled</a>]</div>

{{#if
values.documentStoreEnabled}}
<span class="badge badge-success badge-pill">enabled</span>
{{else}}
<span class="badge badge-warning badge-pill">disabled</span>
{{/if}}
</div>
</li>

<li class="list-group-item d-flex justify-content-between align-items-top">

<div class="form-group form-check mb-0">
<input type="checkbox"
class="form-check-input {{#if errors.documentStoreGenerateEmbeddings}}is-invalid{{/if}}"
id="documentStoreGenerateEmbeddings" name="documentStoreGenerateEmbeddings" {{#if
values.documentStoreGenerateEmbeddings}}checked{{/if}} />
<label class="form-check-label" for="documentStoreGenerateEmbeddings">Enable the "Chat with email"
feature.</label>
{{#if errors.documentStoreGenerateEmbeddings}}
<span class="invalid-feedback">{{errors.documentStoreGenerateEmbeddings}}</span>
{{/if}}
<small class="form-text text-muted">If enabled, then EmailEngine uses OpenAI API to generate text
embeddings for emails stored in the document store. This option requires document store syncing
to be enabled and the OpenAI API key to be set <a href="/admin/config/ai">here</a>. <i>Enabling
this option does not apply to already processed emails.</i></small>
</div>

<div class="text-right">
<div class="text-muted code-link">[<a href="/admin/iframe/docs#/settings/postV1Settings"
target="_blank">documentStoreGenerateEmbeddings</a>]</div>

{{#if
values.documentStoreGenerateEmbeddings}}
<span class="badge badge-success badge-pill">enabled</span>
{{else}}
<span class="badge badge-warning badge-pill">disabled</span>
{{/if}}
</div>
</li>
</ul>

<div class="card-body">

<div class="form-group">
<div class="text-muted float-right code-link">[<a href="/admin/iframe/docs#/settings/postV1Settings"
Expand Down
8 changes: 7 additions & 1 deletion workers/documents.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ const documentsWorker = new Worker(
}
};

for (let indexName of [index, `${index}.threads`]) {
for (let indexName of [index, `${index}.threads`, `${index}.embeddings`]) {
try {
deleteResult[indexName] = await client.deleteByQuery({
index: indexName,
Expand Down Expand Up @@ -552,6 +552,9 @@ const documentsWorker = new Worker(
}
}

// remove from embeddings delete queue
await redis.zrem(`${REDIS_PREFIX}expungequeue`, `${job.data.account}:${messageId}`);

return {
index: indexResult._index,
id: indexResult._id,
Expand Down Expand Up @@ -582,6 +585,9 @@ const documentsWorker = new Worker(
index,
id: `${job.data.account}:${messageId}`
});

// add to embeddings delete queue
await redis.zadd(`${REDIS_PREFIX}expungequeue`, Date.now(), `${job.data.account}:${messageId}`);
} catch (err) {
switch (err.meta && err.meta.body && err.meta.body.result) {
case 'not_found':
Expand Down

0 comments on commit 0c40bd9

Please sign in to comment.