perf: chunk read (#4109)

* package

* perf: chunk read

Author: c121914yu

docSite/content/zh-cn/docs/development/upgrading/491.md

@@ -10,6 +10,7 @@ weight: 799
 ## 🚀 新增内容
 
 1. 商业版支持单团队模式，更好的管理内部成员。
+2. 知识库分块阅读器。
 
 ## ⚙️ 优化
 

packages/global/core/dataset/data/constants.ts

@@ -16,23 +16,23 @@ export const DatasetDataIndexMap: Record<
   }
 > = {
   [DatasetDataIndexTypeEnum.default]: {
-    label: i18nT('dataset:data_index_default'),
+    label: i18nT('common:data_index_default'),
     color: 'gray'
   },
   [DatasetDataIndexTypeEnum.custom]: {
-    label: i18nT('dataset:data_index_custom'),
+    label: i18nT('common:data_index_custom'),
     color: 'blue'
   },
   [DatasetDataIndexTypeEnum.summary]: {
-    label: i18nT('dataset:data_index_summary'),
+    label: i18nT('common:data_index_summary'),
     color: 'green'
   },
   [DatasetDataIndexTypeEnum.question]: {
-    label: i18nT('dataset:data_index_question'),
+    label: i18nT('common:data_index_question'),
     color: 'red'
   },
   [DatasetDataIndexTypeEnum.image]: {
-    label: i18nT('dataset:data_index_image'),
+    label: i18nT('common:data_index_image'),
     color: 'purple'
   }
 };

packages/service/core/chat/saveChat.ts

@@ -74,41 +74,42 @@ export async function saveChat({
       (node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput
     )?.inputs;
 
-    await mongoSessionRun(async (session) => {
-      const processedContent = content.map((item) => {
-        if (item.obj === ChatRoleEnum.AI) {
-          const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse];
+    // Format save chat content: Remove quote q/a
+    const processedContent = content.map((item) => {
+      if (item.obj === ChatRoleEnum.AI) {
+        const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse];
 
-          if (nodeResponse) {
-            return {
-              ...item,
-              [DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => {
-                if (
-                  responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode &&
-                  responseItem.quoteList
-                ) {
-                  return {
-                    ...item,
-                    quoteList: responseItem.quoteList.map((quote: any) => ({
-                      id: quote.id,
-                      chunkIndex: quote.chunkIndex,
-                      datasetId: quote.datasetId,
-                      collectionId: quote.collectionId,
-                      sourceId: quote.sourceId,
-                      sourceName: quote.sourceName,
-                      score: quote.score,
-                      tokens: quote.tokens
-                    }))
-                  };
-                }
-                return item;
-              })
-            };
-          }
+        if (nodeResponse) {
+          return {
+            ...item,
+            [DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => {
+              if (
+                responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode &&
+                responseItem.quoteList
+              ) {
+                return {
+                  ...responseItem,
+                  quoteList: responseItem.quoteList.map((quote: any) => ({
+                    id: quote.id,
+                    chunkIndex: quote.chunkIndex,
+                    datasetId: quote.datasetId,
+                    collectionId: quote.collectionId,
+                    sourceId: quote.sourceId,
+                    sourceName: quote.sourceName,
+                    score: quote.score,
+                    tokens: quote.tokens
+                  }))
+                };
+              }
+              return responseItem;
+            })
+          };
         }
-        return item;
-      });
+      }
+      return item;
+    });
 
+    await mongoSessionRun(async (session) => {
       const [{ _id: chatItemIdHuman }, { _id: chatItemIdAi }] = await MongoChatItem.insertMany(
         processedContent.map((item) => ({
           chatId,

packages/service/core/dataset/data/schema.ts

@@ -98,15 +98,11 @@ try {
     chunkIndex: 1,
     updateTime: -1
   });
-  // FullText tmp full text index
-  // DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
   // Recall vectors after data matching
   DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
   DatasetDataSchema.index({ updateTime: 1 });
   // rebuild data
   DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
-
-  DatasetDataSchema.index({ initFullText: 1 });
 } catch (error) {
   console.log(error);
 }

packages/web/components/common/Icon/icons/common/download.svg

@@ -20,6 +20,7 @@
   "custom_input_guide_url": "Custom Lexicon URL",
   "dataset_quote_type error": "Knowledge base reference type is wrong, correct type: { datasetId: string }[]",
   "delete_all_input_guide_confirm": "Are you sure you want to clear the input guide lexicon?",
+  "download_chunks": "Download data",
   "empty_directory": "This directory is empty~",
   "file_amount_over": "Exceeded maximum file quantity {{max}}",
   "file_input": "File input",
@@ -42,6 +43,7 @@
   "query_extension_IO_tokens": "Problem Optimization Input/Output Tokens",
   "query_extension_result": "Problem optimization results",
   "question_tip": "From top to bottom, the response order of each module",
+  "read_raw_source": "Open the original text",
   "reasoning_text": "Thinking process",
   "response.child total points": "Sub-workflow point consumption",
   "response.dataset_concat_length": "Combined total",
@@ -52,6 +54,7 @@
   "select_img": "Upload Image",
   "source_cronJob": "Scheduled execution",
   "stream_output": "Stream Output",
+  "to_dataset": "Go to the Knowledge Base",
   "unsupported_file_type": "Unsupported file types",
   "upload": "Upload",
   "view_citations": "View References",

packages/web/i18n/en/chat.json

@@ -498,12 +498,9 @@
   "core.dataset.Dataset": "Dataset",
   "core.dataset.Dataset ID": "Dataset ID",
   "core.dataset.Delete Confirm": "Confirm to Delete This Dataset? Data Cannot Be Recovered After Deletion, Please Confirm!",
-  "core.dataset.Download the parsed content": "Download the parsed content",
   "core.dataset.Empty Dataset": "Empty Dataset",
   "core.dataset.Empty Dataset Tips": "No Dataset Yet, Create One Now!",
   "core.dataset.Folder placeholder": "This is a Directory",
-  "core.dataset.Get the raw data": "Get the raw data",
-  "core.dataset.Go Dataset": "Go to Dataset",
   "core.dataset.Intro Placeholder": "This Dataset Has No Introduction Yet",
   "core.dataset.Manual collection": "Manual Dataset",
   "core.dataset.My Dataset": "My Dataset",
@@ -832,6 +829,11 @@
   "core.workflow.variable": "Variable",
   "create": "Create",
   "cron_job_run_app": "Scheduled Task",
+  "data_index_custom": "Custom index",
+  "data_index_default": "Default index",
+  "data_index_image": "Image Index",
+  "data_index_question": "Inferred question index",
+  "data_index_summary": "Summary Index",
   "dataset.Confirm move the folder": "Confirm to Move to This Directory",
   "dataset.Confirm to delete the data": "Confirm to Delete This Data?",
   "dataset.Confirm to delete the file": "Confirm to Delete This File and All Its Data?",

packages/web/i18n/en/common.json

@@ -27,12 +27,7 @@
   "custom_data_process_params_desc": "Customize data processing rules",
   "data.ideal_chunk_length": "ideal block length",
   "data_amount": "{{dataAmount}} Datas, {{indexAmount}} Indexes",
-  "data_index_custom": "Custom index",
-  "data_index_default": "Default index",
-  "data_index_image": "Image Index",
   "data_index_num": "Index {{index}}",
-  "data_index_question": "Inferred question index",
-  "data_index_summary": "Summary index",
   "data_process_params": "Params",
   "data_process_setting": "Processing config",
   "dataset.Unsupported operation": "dataset.Unsupported operation",

packages/web/i18n/en/dataset.json

@@ -20,6 +20,7 @@
   "custom_input_guide_url": "自定义词库地址",
   "dataset_quote_type error": "知识库引用类型错误，正确类型：{ datasetId: string }[]",
   "delete_all_input_guide_confirm": "确定要清空输入引导词库吗？",
+  "download_chunks": "下载数据",
   "empty_directory": "这个目录已经没东西可选了~",
   "file_amount_over": "超出最大文件数量 {{max}}",
   "file_input": "系统文件",
@@ -42,6 +43,7 @@
   "query_extension_IO_tokens": "问题优化输入/输出 Tokens",
   "query_extension_result": "问题优化结果",
   "question_tip": "从上到下，为各个模块的响应顺序",
+  "read_raw_source": "打开原文",
   "reasoning_text": "思考过程",
   "response.child total points": "子工作流积分消耗",
   "response.dataset_concat_length": "合并后总数",
@@ -52,6 +54,7 @@
   "select_img": "上传图片",
   "source_cronJob": "定时执行",
   "stream_output": "流输出",
+  "to_dataset": "前往知识库",
   "unsupported_file_type": "不支持的文件类型",
   "upload": "上传",
   "view_citations": "查看引用",

packages/web/i18n/zh-CN/chat.json

@@ -501,12 +501,9 @@
   "core.dataset.Dataset": "知识库",
   "core.dataset.Dataset ID": "知识库 ID",
   "core.dataset.Delete Confirm": "确认删除该知识库？删除后数据无法恢复，请确认！",
-  "core.dataset.Download the parsed content": "下载解析内容",
   "core.dataset.Empty Dataset": "空数据集",
   "core.dataset.Empty Dataset Tips": "还没有知识库，快去创建一个吧！",
   "core.dataset.Folder placeholder": "这是一个目录",
-  "core.dataset.Get the raw data": "获取源数据",
-  "core.dataset.Go Dataset": "前往知识库",
   "core.dataset.Intro Placeholder": "这个知识库还没有介绍~",
   "core.dataset.Manual collection": "手动数据集",
   "core.dataset.My Dataset": "我的知识库",
@@ -836,6 +833,11 @@
   "core.workflow.variable": "变量",
   "create": "去创建",
   "cron_job_run_app": "定时任务",
+  "data_index_custom": "自定义索引",
+  "data_index_default": "默认索引",
+  "data_index_image": "图片索引",
+  "data_index_question": "推测问题索引",
+  "data_index_summary": "摘要索引",
   "dataset.Confirm move the folder": "确认移动到该目录",
   "dataset.Confirm to delete the data": "确认删除该数据？",
   "dataset.Confirm to delete the file": "确认删除该文件及其所有数据？",