diff --git a/deploy/docker/docker-compose-milvus.yml b/deploy/docker/docker-compose-milvus.yml index e5a90446bd22..0ba7a90122b7 100644 --- a/deploy/docker/docker-compose-milvus.yml +++ b/deploy/docker/docker-compose-milvus.yml @@ -100,7 +100,7 @@ services: exec docker-entrypoint.sh "$$@" & # 等待MongoDB服务启动 - until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')" > /dev/null 2>&1; do + until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do echo "Waiting for MongoDB to start..." sleep 2 done @@ -175,7 +175,7 @@ services: # AI Proxy aiproxy: - image: 'ghcr.io/labring/sealos-aiproxy-service:latest' + image: ghcr.io/labring/sealos-aiproxy-service:latest container_name: aiproxy restart: unless-stopped depends_on: @@ -191,7 +191,7 @@ services: # 数据库连接地址 - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy # 最大重试次数 - - RetryTimes=3 + - RETRY_TIMES=3 # 不需要计费 - BILLING_ENABLED=false # 不需要严格检测模型 diff --git a/deploy/docker/docker-compose-pgvector.yml b/deploy/docker/docker-compose-pgvector.yml index 0d763c912cd5..b4bb21a3a394 100644 --- a/deploy/docker/docker-compose-pgvector.yml +++ b/deploy/docker/docker-compose-pgvector.yml @@ -58,7 +58,7 @@ services: exec docker-entrypoint.sh "$$@" & # 等待MongoDB服务启动 - until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')" > /dev/null 2>&1; do + until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do echo "Waiting for MongoDB to start..." sleep 2 done @@ -132,7 +132,7 @@ services: # AI Proxy aiproxy: - image: 'ghcr.io/labring/sealos-aiproxy-service:latest' + image: ghcr.io/labring/sealos-aiproxy-service:latest container_name: aiproxy restart: unless-stopped depends_on: @@ -148,7 +148,7 @@ services: # 数据库连接地址 - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy # 最大重试次数 - - RetryTimes=3 + - RETRY_TIMES=3 # 不需要计费 - BILLING_ENABLED=false # 不需要严格检测模型 diff --git a/deploy/docker/docker-compose-zilliz.yml b/deploy/docker/docker-compose-zilliz.yml index c4cafe71cf50..23cad557f7c0 100644 --- a/deploy/docker/docker-compose-zilliz.yml +++ b/deploy/docker/docker-compose-zilliz.yml @@ -41,7 +41,7 @@ services: exec docker-entrypoint.sh "$$@" & # 等待MongoDB服务启动 - until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')" > /dev/null 2>&1; do + until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do echo "Waiting for MongoDB to start..." sleep 2 done @@ -113,7 +113,7 @@ services: # AI Proxy aiproxy: - image: 'ghcr.io/labring/sealos-aiproxy-service:latest' + image: ghcr.io/labring/sealos-aiproxy-service:latest container_name: aiproxy restart: unless-stopped depends_on: @@ -129,7 +129,7 @@ services: # 数据库连接地址 - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy # 最大重试次数 - - RetryTimes=3 + - RETRY_TIMES=3 # 不需要计费 - BILLING_ENABLED=false # 不需要严格检测模型 diff --git a/docSite/content/zh-cn/docs/development/upgrading/4823.md b/docSite/content/zh-cn/docs/development/upgrading/4823.md index 446245b1eb2f..4741613a3318 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/4823.md +++ b/docSite/content/zh-cn/docs/development/upgrading/4823.md @@ -4,7 +4,7 @@ description: 'FastGPT V4.8.23 更新说明' icon: 'upgrade' draft: false toc: true -weight: 802 +weight: 801 --- ## 更新指南 diff --git a/docSite/content/zh-cn/docs/development/upgrading/490.md b/docSite/content/zh-cn/docs/development/upgrading/490.md index eb654c1b506e..b61bf0f0a453 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/490.md +++ b/docSite/content/zh-cn/docs/development/upgrading/490.md @@ -4,7 +4,7 @@ description: 'FastGPT V4.9.0 更新说明' icon: 'upgrade' draft: false toc: true -weight: 801 +weight: 800 --- diff --git a/docSite/content/zh-cn/docs/development/upgrading/491.md b/docSite/content/zh-cn/docs/development/upgrading/491.md new file mode 100644 index 000000000000..2fd382afe2eb --- /dev/null +++ b/docSite/content/zh-cn/docs/development/upgrading/491.md @@ -0,0 +1,24 @@ +--- +title: 'V4.9.1' +description: 'FastGPT V4.9.1 更新说明' +icon: 'upgrade' +draft: false +toc: true +weight: 799 +--- + +## 🚀 新增内容 + + +## ⚙️ 优化 + +1. 知识库数据输入框交互 +2. 应用拉取绑定知识库数据交由后端处理。 +3. 增加依赖包安全版本检测，并升级部分依赖包。 + +## 🐛 修复 + +1. 最大响应 tokens 提示显示错误的问题。 +2. HTTP Node 中，字符串包含换行符时，会解析失败。 +3. 知识库问题优化中，未传递历史记录。 +4. 错误提示翻译缺失。 \ No newline at end of file diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts index c88023213065..fe1212ba7ea8 100644 --- a/packages/global/common/string/textSplitter.ts +++ b/packages/global/common/string/textSplitter.ts @@ -93,7 +93,7 @@ ${mdSplitString} /* 1. 自定义分隔符：不需要重叠，不需要小块合并 - 2. Markdown 标题：不需要重叠；标题嵌套共享，不需要小块合并 + 2. Markdown 标题：不需要重叠；标题嵌套共享，需要小块合并 3. 特殊 markdown 语法：不需要重叠，需要小块合并 4. 段落：尽可能保证它是一个完整的段落。 5. 标点分割：重叠 @@ -227,7 +227,7 @@ const commonSplit = (props: SplitProps): SplitResponse => { }): string[] => { const isMarkdownStep = checkIsMarkdownSplit(step); const isCustomStep = checkIsCustomStep(step); - const forbidConcat = isMarkdownStep || isCustomStep; // forbid=true时候，lastText肯定为空 + const forbidConcat = isCustomStep; // forbid=true时候，lastText肯定为空 // oversize if (step >= stepReges.length) { diff --git a/packages/service/common/vectorStore/pg/class.ts b/packages/service/common/vectorStore/pg/class.ts index 6b7f42bd3ff7..26db89074c16 100644 --- a/packages/service/common/vectorStore/pg/class.ts +++ b/packages/service/common/vectorStore/pg/class.ts @@ -38,6 +38,27 @@ export class PgVectorCtrl { await PgClient.query( `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${DatasetVectorTableName} USING btree(createtime);` ); + // 10w rows + // await PgClient.query(` + // ALTER TABLE modeldata SET ( + // autovacuum_vacuum_scale_factor = 0.1, + // autovacuum_analyze_scale_factor = 0.05, + // autovacuum_vacuum_threshold = 50, + // autovacuum_analyze_threshold = 50, + // autovacuum_vacuum_cost_delay = 20, + // autovacuum_vacuum_cost_limit = 200 + // );`); + + // 100w rows + // await PgClient.query(` + // ALTER TABLE modeldata SET ( + // autovacuum_vacuum_scale_factor = 0.01, + // autovacuum_analyze_scale_factor = 0.02, + // autovacuum_vacuum_threshold = 1000, + // autovacuum_analyze_threshold = 1000, + // autovacuum_vacuum_cost_delay = 10, + // autovacuum_vacuum_cost_limit = 2000 + // );`) addLog.info('init pg successful'); } catch (error) { diff --git a/packages/service/core/app/utils.ts b/packages/service/core/app/utils.ts index a5504e8e860a..99b520fb65f1 100644 --- a/packages/service/core/app/utils.ts +++ b/packages/service/core/app/utils.ts @@ -4,20 +4,13 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node'; -export type ListByAppIdAndDatasetIdsBody = { - teamId: string; - datasetIdList: string[]; -}; - -interface Dataset { - datasetId: string; - [key: string]: any; -} - export async function listAppDatasetDataByTeamIdAndDatasetIds({ teamId, datasetIdList -}: ListByAppIdAndDatasetIdsBody) { +}: { + teamId?: string; + datasetIdList: string[]; +}) { const myDatasets = await MongoDataset.find({ teamId, _id: { $in: datasetIdList } @@ -31,9 +24,18 @@ export async function listAppDatasetDataByTeamIdAndDatasetIds({ })); } -export async function rewriteAppWorkflowToDetail(nodes: StoreNodeItemType[], teamId: string) { +export async function rewriteAppWorkflowToDetail({ + nodes, + teamId, + isRoot +}: { + nodes: StoreNodeItemType[]; + teamId: string; + isRoot: boolean; +}) { const datasetIdSet = new Set(); + // Get all dataset ids from nodes nodes.forEach((node) => { if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return; @@ -41,83 +43,83 @@ export async function rewriteAppWorkflowToDetail(nodes: StoreNodeItemType[], tea if (!input) return; const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string }; + if (!rawValue) return; const datasetIds = Array.isArray(rawValue) - ? rawValue - .map((v) => v?.datasetId) - .filter((id): id is string => !!id && typeof id === 'string') - : rawValue?.datasetId + ? rawValue.map((v) => v?.datasetId).filter((id) => !!id && typeof id === 'string') + : rawValue.datasetId ? [String(rawValue.datasetId)] : []; - if (datasetIds.length === 0) return; - datasetIds.forEach((id) => datasetIdSet.add(id)); }); if (datasetIdSet.size === 0) return; - const uniqueDatasetIds = Array.from(datasetIdSet); + // Load dataset list const datasetList = await listAppDatasetDataByTeamIdAndDatasetIds({ - teamId, - datasetIdList: uniqueDatasetIds + teamId: isRoot ? undefined : teamId, + datasetIdList: Array.from(datasetIdSet) }); + const datasetMap = new Map(datasetList.map((ds) => [String(ds.datasetId), ds])); - const datasetMap = new Map( - datasetList.map((ds) => [ - String(ds.datasetId), - { - ...ds, - vectorModel: getEmbeddingModel(ds.vectorModel.model) - } - ]) - ); - + // Rewrite dataset ids, add dataset info to nodes nodes.forEach((node) => { if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return; - const input = node.inputs.find((item) => item.key === NodeInputKeyEnum.datasetSelectList); - if (!input) return; - - const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string }; - - const datasetIds = Array.isArray(rawValue) - ? rawValue - .map((v) => v?.datasetId) - .filter((id): id is string => !!id && typeof id === 'string') - : rawValue?.datasetId - ? [String(rawValue.datasetId)] - : []; - - if (datasetIds.length === 0) return; - - input.value = datasetIds - .map((id) => { - const data = datasetMap.get(String(id)); - return data - ? { - datasetId: data.datasetId, - avatar: data.avatar, - name: data.name, - vectorModel: data.vectorModel - } - : undefined; - }) - .filter((item): item is NonNullable => !!item); + node.inputs.forEach((item) => { + if (item.key !== NodeInputKeyEnum.datasetSelectList) return; + + const val = item.value as undefined | { datasetId: string }[] | { datasetId: string }; + + if (Array.isArray(val)) { + item.value = val.map((v) => { + const data = datasetMap.get(String(v.datasetId))!; + return { + datasetId: data.datasetId, + avatar: data.avatar, + name: data.name, + vectorModel: data.vectorModel + }; + }); + } else if (typeof val === 'object' && val !== null) { + const data = datasetMap.get(String(val.datasetId))!; + item.value = { + datasetId: data.datasetId, + avatar: data.avatar, + name: data.name, + vectorModel: data.vectorModel + }; + } + }); }); + + return nodes; } export async function rewriteAppWorkflowToSimple(formatNodes: StoreNodeItemType[]) { formatNodes.forEach((node) => { if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return; - const datasetsInput = node.inputs.find( - (input) => input.key === NodeInputKeyEnum.datasetSelectList - ); - if (datasetsInput?.value) { - datasetsInput.value = datasetsInput.value.map((dataset: Dataset) => ({ - datasetId: dataset.datasetId - })); - } + node.inputs.forEach((input) => { + if (input.key === NodeInputKeyEnum.datasetSelectList) { + const val = input.value as undefined | { datasetId: string }[] | { datasetId: string }; + if (!val) { + input.value = []; + } else if (Array.isArray(val)) { + input.value = val + .map((dataset: { datasetId: string }) => ({ + datasetId: dataset.datasetId + })) + .filter((item) => !!item.datasetId); + } else if (typeof val === 'object' && val !== null) { + input.value = [ + { + datasetId: val.datasetId + } + ]; + } + } + }); }); } diff --git a/packages/web/components/common/Icon/button.tsx b/packages/web/components/common/Icon/button.tsx index b335a117a0e2..7a6bc8eedf77 100644 --- a/packages/web/components/common/Icon/button.tsx +++ b/packages/web/components/common/Icon/button.tsx @@ -34,6 +34,7 @@ const MyIconButton = ({ color: hoverColor }} onClick={onClick} + sx={{ userSelect: 'none' }} {...props} > diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 3d1d713b963f..5511c1fb10e7 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -373,9 +373,13 @@ export const iconPaths = { 'modal/teamPlans': () => import('./icons/modal/teamPlans.svg'), 'model/BAAI': () => import('./icons/model/BAAI.svg'), 'model/alicloud': () => import('./icons/model/alicloud.svg'), + 'model/aws': () => import('./icons/model/aws.svg'), + 'model/azure': () => import('./icons/model/azure.svg'), 'model/baichuan': () => import('./icons/model/baichuan.svg'), 'model/chatglm': () => import('./icons/model/chatglm.svg'), 'model/claude': () => import('./icons/model/claude.svg'), + 'model/cloudflare': () => import('./icons/model/cloudflare.svg'), + 'model/cohere': () => import('./icons/model/cohere.svg'), 'model/deepseek': () => import('./icons/model/deepseek.svg'), 'model/doubao': () => import('./icons/model/doubao.svg'), 'model/ernie': () => import('./icons/model/ernie.svg'), diff --git a/packages/web/components/common/Icon/icons/core/chat/chevronDown.svg b/packages/web/components/common/Icon/icons/core/chat/chevronDown.svg index cc8ed229cf08..d69e3d30b4f5 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chevronDown.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chevronDown.svg @@ -1,3 +1,3 @@ - - - + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chevronUp.svg b/packages/web/components/common/Icon/icons/core/chat/chevronUp.svg index 9088fc25812a..34de47bde169 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chevronUp.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chevronUp.svg @@ -1,3 +1,3 @@ - - - + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/aws.svg b/packages/web/components/common/Icon/icons/model/aws.svg new file mode 100644 index 000000000000..39e8331497c4 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/aws.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/azure.svg b/packages/web/components/common/Icon/icons/model/azure.svg new file mode 100644 index 000000000000..b93c3ea6b857 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/azure.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/cloudflare.svg b/packages/web/components/common/Icon/icons/model/cloudflare.svg new file mode 100644 index 000000000000..839f98610898 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/cloudflare.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/model/cohere.svg b/packages/web/components/common/Icon/icons/model/cohere.svg new file mode 100644 index 000000000000..9fd187431b92 --- /dev/null +++ b/packages/web/components/common/Icon/icons/model/cohere.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index d71060c08059..0e4cc97c7ade 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -534,16 +534,10 @@ "core.dataset.collection.status.active": "Ready", "core.dataset.collection.sync.result.sameRaw": "Content Unchanged, No Update Needed", "core.dataset.collection.sync.result.success": "Sync Started", - "core.dataset.data.Auxiliary Data": "Auxiliary Data", - "core.dataset.data.Auxiliary Data Placeholder": "This part is optional and is usually used to construct structured prompts in conjunction with the 'Data Content' above for special scenarios, up to {{maxToken}} characters.", - "core.dataset.data.Auxiliary Data Tip": "This part is optional\nThis content is usually used to construct structured prompts in conjunction with the data content above for special scenarios", "core.dataset.data.Data Content": "Related Data Content", - "core.dataset.data.Data Content Placeholder": "This input box is required. This content is usually a description of the knowledge point or a user's question, up to {{maxToken}} characters.", - "core.dataset.data.Data Content Tip": "This input box is required\nThis content is usually a description of the knowledge point or a user's question.", "core.dataset.data.Default Index Tip": "Cannot be edited. The default index will use the text of 'Related Data Content' and 'Auxiliary Data' to generate the index directly.", "core.dataset.data.Edit": "Edit Data", "core.dataset.data.Empty Tip": "This collection has no data yet", - "core.dataset.data.Main Content": "Main Content", "core.dataset.data.Search data placeholder": "Search Related Data", "core.dataset.data.Too Long": "Total Length Exceeded", "core.dataset.data.group": "Group", @@ -867,6 +861,12 @@ "dataset.dataset_name": "Dataset Name", "dataset.deleteFolderTips": "Confirm to Delete This Folder and All Its Contained Datasets? Data Cannot Be Recovered After Deletion, Please Confirm!", "dataset.test.noResult": "No Search Results", + "dataset_data_import_q_placeholder": "Up to {{maxToken}} words.", + "dataset_data_input_a": "Answer", + "dataset_data_input_chunk": "Chunk", + "dataset_data_input_chunk_content": "Chunk", + "dataset_data_input_q": "question", + "dataset_data_input_qa": "QA", "dataset_text_model_tip": "Used for text processing in the knowledge base preprocessing stage, such as automatic supplementary indexing, Q&A pair extraction.", "deep_rag_search": "In-depth search", "delete_api": "Are you sure you want to delete this API key? \nAfter deletion, the key will become invalid immediately and the corresponding conversation log will not be deleted. Please confirm!", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 0902a421ed41..0b9a951c8998 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -69,6 +69,7 @@ "import_model_config": "Model selection", "import_param_setting": "Parameter settings", "import_select_file": "Select a file", + "import_select_link": "Enter link", "is_open_schedule": "Enable scheduled synchronization", "keep_image": "Keep the picture", "move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index 47821542d1bc..bc119e67cf40 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -537,16 +537,10 @@ "core.dataset.collection.status.active": "已就绪", "core.dataset.collection.sync.result.sameRaw": "内容未变动，无需更新", "core.dataset.collection.sync.result.success": "开始同步", - "core.dataset.data.Auxiliary Data": "辅助数据", - "core.dataset.data.Auxiliary Data Placeholder": "该部分为可选填项，通常是为了与前面的【数据内容】配合，构建结构化提示词，用于特殊场景，最多 {{maxToken}} 字。", - "core.dataset.data.Auxiliary Data Tip": "该部分为可选填项\n该内容通常是为了与前面的数据内容配合，构建结构化提示词，用于特殊场景", "core.dataset.data.Data Content": "相关数据内容", - "core.dataset.data.Data Content Placeholder": "该输入框是必填项，该内容通常是对于知识点的描述，也可以是用户的问题，最多 {{maxToken}} 字。", - "core.dataset.data.Data Content Tip": "该输入框是必填项\n该内容通常是对于知识点的描述，也可以是用户的问题。", "core.dataset.data.Default Index Tip": "无法编辑，默认索引会使用【相关数据内容】与【辅助数据】的文本直接生成索引。", "core.dataset.data.Edit": "编辑数据", "core.dataset.data.Empty Tip": "这个集合还没有数据~", - "core.dataset.data.Main Content": "主要内容", "core.dataset.data.Search data placeholder": "搜索相关数据", "core.dataset.data.Too Long": "总长度超长了", "core.dataset.data.group": "组", @@ -560,6 +554,7 @@ "core.dataset.error.unAuthDatasetData": "无权操作该数据", "core.dataset.error.unAuthDatasetFile": "无权操作该文件", "core.dataset.error.unCreateCollection": "无权操作该数据", + "core.dataset.error.unExistDataset": "数据集不存在", "core.dataset.error.unLinkCollection": "不是网络链接集合", "core.dataset.externalFile": "外部文件库", "core.dataset.file": "文件", @@ -664,7 +659,6 @@ "core.dataset.website.Selector Course": "使用教程", "core.dataset.website.Start Sync": "开始同步", "core.dataset.website.UnValid Website Tip": "您的站点可能非静态站点，无法同步", - "core.dataset.error.unExistDataset": "数据集不存在", "core.module.Add question type": "添加问题类型", "core.module.Add_option": "添加选项", "core.module.Can not connect self": "不能连接自身", @@ -871,6 +865,12 @@ "dataset.dataset_name": "知识库名称", "dataset.deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！", "dataset.test.noResult": "搜索结果为空", + "dataset_data_import_q_placeholder": "最多 {{maxToken}} 字。", + "dataset_data_input_a": "答案", + "dataset_data_input_chunk": "常规模式", + "dataset_data_input_chunk_content": "内容", + "dataset_data_input_q": "问题", + "dataset_data_input_qa": "QA 模式", "dataset_text_model_tip": "用于知识库预处理阶段的文本处理，例如自动补充索引、问答对提取。", "deep_rag_search": "深度搜索", "delete_api": "确认删除该API密钥？删除后该密钥立即失效，对应的对话日志不会删除，请确认！", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index 2dc1cc54cc00..064b5a73bb91 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -69,6 +69,7 @@ "import_model_config": "模型选择", "import_param_setting": "参数设置", "import_select_file": "选择文件", + "import_select_link": "输入链接", "is_open_schedule": "启用定时同步", "keep_image": "保留图片", "move.hint": "移动后，所选知识库/文件夹将继承新文件夹的权限设置，原先的权限设置失效。", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index e3fd6e908f54..4fb5a389c0e0 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -533,16 +533,10 @@ "core.dataset.collection.status.active": "已就緒", "core.dataset.collection.sync.result.sameRaw": "內容未變更，無需更新", "core.dataset.collection.sync.result.success": "開始同步", - "core.dataset.data.Auxiliary Data": "輔助資料", - "core.dataset.data.Auxiliary Data Placeholder": "此部分為選填項目，通常是為了與前面的【資料內容】配合，建構結構化提示詞，用於特殊場景，最多 {{maxToken}} 字。", - "core.dataset.data.Auxiliary Data Tip": "此部分為選填項目\n此內容通常是為了與前面的資料內容配合，建構結構化提示詞，用於特殊場景", "core.dataset.data.Data Content": "相關資料內容", - "core.dataset.data.Data Content Placeholder": "此輸入欄位為必填項目，此內容通常是知識點的描述，也可以是使用者的問題，最多 {{maxToken}} 字。", - "core.dataset.data.Data Content Tip": "此輸入欄位為必填項目\n此內容通常是知識點的描述，也可以是使用者的問題。", "core.dataset.data.Default Index Tip": "無法編輯，預設索引會使用【相關資料內容】與【輔助資料】的文字直接產生索引。", "core.dataset.data.Edit": "編輯資料", "core.dataset.data.Empty Tip": "此集合還沒有資料", - "core.dataset.data.Main Content": "主要內容", "core.dataset.data.Search data placeholder": "搜尋相關資料", "core.dataset.data.Too Long": "總長度超出上限", "core.dataset.data.group": "組", @@ -867,6 +861,12 @@ "dataset.dataset_name": "知識庫名稱", "dataset.deleteFolderTips": "確認刪除此資料夾及其包含的所有知識庫？刪除後資料無法復原，請確認！", "dataset.test.noResult": "搜尋結果為空", + "dataset_data_import_q_placeholder": "最多 {{maxToken}} 字。", + "dataset_data_input_a": "答案", + "dataset_data_input_chunk": "常規模式", + "dataset_data_input_chunk_content": "內容", + "dataset_data_input_q": "問題", + "dataset_data_input_qa": "QA 模式", "dataset_text_model_tip": "用於知識庫預處理階段的文本處理，例如自動補充索引、問答對提取。", "deep_rag_search": "深度搜索", "delete_api": "確認刪除此 API 金鑰？\n刪除後該金鑰將立即失效，對應的對話記錄不會被刪除，請確認！", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index 7eab0f0805db..706714283744 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -69,6 +69,7 @@ "import_model_config": "模型選擇", "import_param_setting": "參數設置", "import_select_file": "選擇文件", + "import_select_link": "輸入鏈接", "is_open_schedule": "啟用定時同步", "keep_image": "保留圖片", "move.hint": "移動後，所選資料集／資料夾將繼承新資料夾的權限設定，原先的權限設定將失效。", diff --git a/projects/app/src/global/aiproxy/constants.ts b/projects/app/src/global/aiproxy/constants.ts index 5fb0d35c0ec6..5093fa664ef2 100644 --- a/projects/app/src/global/aiproxy/constants.ts +++ b/projects/app/src/global/aiproxy/constants.ts @@ -40,12 +40,16 @@ export const defaultChannel: ChannelInfoType = { priority: 0 }; -export const aiproxyIdMap: Record = { +export const aiproxyIdMap: Record< + number, + { label: string; provider: ModelProviderIdType; avatar?: string } +> = { 1: { label: 'OpenAI', provider: 'OpenAI' }, 3: { + avatar: 'model/azure', label: i18nT('account_model:azure'), provider: 'OpenAI' }, @@ -124,5 +128,28 @@ export const aiproxyIdMap: Record { const res = providerList.find((item) => item.value === providerType); return res; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx b/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx index 853efddecd3c..d437f39c1d55 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/Context.tsx @@ -113,7 +113,7 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode ], [ImportDataSourceEnum.fileLink]: [ { - title: t('dataset:import_select_file') + title: t('dataset:import_select_link') }, { title: t('dataset:import_param_setting') diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx index dee1dd83fa04..a8facf7eec3f 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx @@ -21,6 +21,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); +const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); const APIDatasetCollection = () => { const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); @@ -29,7 +30,8 @@ const APIDatasetCollection = () => { <> {activeStep === 0 && } {activeStep === 1 && } - {activeStep === 2 && } + {activeStep === 2 && } + {activeStep === 3 && } ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ExternalFile.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ExternalFile.tsx index e1e29b00acf8..e1a6beb23705 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ExternalFile.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ExternalFile.tsx @@ -27,6 +27,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); +const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); const ExternalFileCollection = () => { const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); @@ -35,7 +36,8 @@ const ExternalFileCollection = () => { <> {activeStep === 0 && } {activeStep === 1 && } - {activeStep === 2 && } + {activeStep === 2 && } + {activeStep === 3 && } ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileCustomText.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileCustomText.tsx index 12c0c28de428..fcf08248d6c4 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileCustomText.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileCustomText.tsx @@ -13,6 +13,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); +const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); const CustomTet = () => { const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); @@ -20,7 +21,8 @@ const CustomTet = () => { <> {activeStep === 0 && } {activeStep === 1 && } - {activeStep === 2 && } + {activeStep === 2 && } + {activeStep === 3 && } ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLink.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLink.tsx index b9f0e192ea00..ca41cf40cfae 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLink.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLink.tsx @@ -16,6 +16,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); +const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); const LinkCollection = () => { const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); @@ -24,7 +25,8 @@ const LinkCollection = () => { <> {activeStep === 0 && } {activeStep === 1 && } - {activeStep === 2 && } + {activeStep === 2 && } + {activeStep === 3 && } ); }; diff --git a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx index ba7d56fa2e10..068e0ea64e00 100644 --- a/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx +++ b/projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx @@ -57,7 +57,7 @@ const ReTraining = () => { qaChunkSize: collection.chunkSize, customSplitChar: collection.chunkSplitter, qaPrompt: collection.qaPrompt, - webSelector: collection.metadata?.webSelector + webSelector: collection.metadata?.webPageSelector }); } }); diff --git a/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx b/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx index ce69dea0128f..bbb9a73a0156 100644 --- a/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx @@ -1,12 +1,6 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { Box, Flex, Button, Textarea } from '@chakra-ui/react'; -import { - FieldArrayWithId, - UseFieldArrayRemove, - UseFormRegister, - useFieldArray, - useForm -} from 'react-hook-form'; +import { Box, Flex, Button, Textarea, ModalFooter, HStack, VStack } from '@chakra-ui/react'; +import { UseFormRegister, useFieldArray, useForm } from 'react-hook-form'; import { postInsertData2Dataset, putDatasetDataById, @@ -17,7 +11,6 @@ import { useToast } from '@fastgpt/web/hooks/useToast'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyModal from '@fastgpt/web/components/common/MyModal'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; -import { useQuery } from '@tanstack/react-query'; import { useTranslation } from 'next-i18next'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; @@ -28,25 +21,27 @@ import { getDocPath } from '@/web/common/system/doc'; import MyBox from '@fastgpt/web/components/common/MyBox'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useSystemStore } from '@/web/common/system/useSystemStore'; -import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; -import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs'; import styles from './styles.module.scss'; import { DatasetDataIndexTypeEnum, getDatasetIndexMapData } from '@fastgpt/global/core/dataset/data/constants'; +import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs'; +import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; +import MyIconButton from '@fastgpt/web/components/common/Icon/button'; export type InputDataType = { q: string; a: string; indexes: (Omit & { dataId?: string; // pg data id + fold: boolean; })[]; }; enum TabEnum { - content = 'content', - index = 'index' + chunk = 'chunk', + qa = 'qa' } const InputDataModal = ({ @@ -64,73 +59,47 @@ const InputDataModal = ({ }) => { const { t } = useTranslation(); const { toast } = useToast(); - const [currentTab, setCurrentTab] = useState(TabEnum.content); const { embeddingModelList, defaultModels } = useSystemStore(); + const [currentTab, setCurrentTab] = useState(TabEnum.chunk); + const { register, handleSubmit, reset, control } = useForm(); const { fields: indexes, - append: appendIndexes, - remove: removeIndexes + prepend: prependIndexes, + remove: removeIndexes, + update: updateIndexes } = useFieldArray({ control, name: 'indexes' }); - const tabList = [ - { - label: ( - - {t('common:dataset.data.edit.divide_content')} - - ), - value: TabEnum.content - }, - { - label: ( - - {t('common:dataset.data.edit.Index', { amount: indexes.length })} - - - window.open(getDocPath('/docs/guide/knowledge_base/dataset_engine/'), '_blank') - } - _hover={{ - color: 'primary.600', - cursor: 'pointer' - }} - /> - - - ), - value: TabEnum.index - } - ]; - - const { data: collection = defaultCollectionDetail } = useQuery( - ['loadCollectionId', collectionId], + const { data: collection = defaultCollectionDetail } = useRequest2( () => { return getDatasetCollectionById(collectionId); + }, + { + manual: false, + refreshDeps: [collectionId] } ); - const { isFetching: isFetchingData } = useQuery( - ['getDatasetDataItemById', dataId], - () => { + const { loading: isFetchingData } = useRequest2( + async () => { if (dataId) return getDatasetDataItemById(dataId); return null; }, { + manual: false, + refreshDeps: [dataId], onSuccess(res) { if (res) { reset({ q: res.q, a: res.a, - indexes: res.indexes + indexes: res.indexes.map((item) => ({ + ...item, + fold: true + })) }); } else if (defaultValue) { reset({ @@ -138,6 +107,10 @@ const InputDataModal = ({ a: defaultValue.a }); } + + if (res?.a || defaultValue?.a) { + setCurrentTab(TabEnum.qa); + } }, onError(err) { toast({ @@ -161,7 +134,6 @@ const InputDataModal = ({ const { runAsync: sureImportData, loading: isImporting } = useRequest2( async (e: InputDataType) => { if (!e.q) { - setCurrentTab(TabEnum.content); return Promise.reject(t('common:dataset.data.input is empty')); } @@ -175,9 +147,9 @@ const InputDataModal = ({ const dataId = await postInsertData2Dataset({ collectionId: collection._id, q: e.q, - a: e.a, + a: currentTab === TabEnum.qa ? e.a : '', // Contains no default index - indexes: e.indexes + indexes: e.indexes.filter((item) => !!item.text?.trim()) }); return { @@ -186,6 +158,7 @@ const InputDataModal = ({ }; }, { + refreshDeps: [currentTab], successToast: t('common:dataset.data.Input Success Tip'), onSuccess(e) { reset({ @@ -208,8 +181,9 @@ const InputDataModal = ({ // not exactly same await putDatasetDataById({ dataId, - ...e, - indexes: e.indexes + q: e.q, + a: currentTab === TabEnum.qa ? e.a : '', + indexes: e.indexes.filter((item) => !!item.text?.trim()) }); return { @@ -218,6 +192,7 @@ const InputDataModal = ({ }; }, { + refreshDeps: [currentTab], successToast: t('common:dataset.data.Update Success Tip'), onSuccess(data) { onSuccess(data); @@ -267,49 +242,166 @@ const InputDataModal = ({ isLoading={isLoading} h={'100%'} py={[6, '1.5rem']} - px={[5, '3.25rem']} > - - - - list={tabList} - p={0} - value={currentTab} - onChange={(e: TabEnum) => setCurrentTab(e)} - /> - - {currentTab === TabEnum.index && ( -