fix: import dataset step error;perf: ai proxy avatar

deploy/docker/docker-compose-milvus.yml

@@ -100,7 +100,7 @@ services:
         exec docker-entrypoint.sh "$$@" &
 
         # 等待MongoDB服务启动
-        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')" > /dev/null 2>&1; do
+        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do
           echo "Waiting for MongoDB to start..."
           sleep 2
         done
@@ -175,7 +175,7 @@ services:
 
   # AI Proxy
   aiproxy:
-    image: 'ghcr.io/labring/sealos-aiproxy-service:latest'
+    image: ghcr.io/labring/sealos-aiproxy-service:latest
     container_name: aiproxy
     restart: unless-stopped
     depends_on:
@@ -191,7 +191,7 @@ services:
       # 数据库连接地址
       - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy
       # 最大重试次数
-      - RetryTimes=3
+      - RETRY_TIMES=3
       # 不需要计费
       - BILLING_ENABLED=false
       # 不需要严格检测模型

deploy/docker/docker-compose-pgvector.yml

@@ -58,7 +58,7 @@ services:
         exec docker-entrypoint.sh "$$@" &
 
         # 等待MongoDB服务启动
-        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')" > /dev/null 2>&1; do
+        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do
           echo "Waiting for MongoDB to start..."
           sleep 2
         done
@@ -132,7 +132,7 @@ services:
 
   # AI Proxy
   aiproxy:
-    image: 'ghcr.io/labring/sealos-aiproxy-service:latest'
+    image: ghcr.io/labring/sealos-aiproxy-service:latest
     container_name: aiproxy
     restart: unless-stopped
     depends_on:
@@ -148,7 +148,7 @@ services:
       # 数据库连接地址
       - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy
       # 最大重试次数
-      - RetryTimes=3
+      - RETRY_TIMES=3
       # 不需要计费
       - BILLING_ENABLED=false
       # 不需要严格检测模型

deploy/docker/docker-compose-zilliz.yml

@@ -41,7 +41,7 @@ services:
         exec docker-entrypoint.sh "$$@" &
 
         # 等待MongoDB服务启动
-        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')" > /dev/null 2>&1; do
+        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do
           echo "Waiting for MongoDB to start..."
           sleep 2
         done
@@ -113,7 +113,7 @@ services:
 
   # AI Proxy
   aiproxy:
-    image: 'ghcr.io/labring/sealos-aiproxy-service:latest'
+    image: ghcr.io/labring/sealos-aiproxy-service:latest
     container_name: aiproxy
     restart: unless-stopped
     depends_on:
@@ -129,7 +129,7 @@ services:
       # 数据库连接地址
       - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy
       # 最大重试次数
-      - RetryTimes=3
+      - RETRY_TIMES=3
       # 不需要计费
       - BILLING_ENABLED=false
       # 不需要严格检测模型

docSite/content/zh-cn/docs/development/upgrading/4823.md

@@ -4,7 +4,7 @@ description: 'FastGPT V4.8.23 更新说明'
 icon: 'upgrade'
 draft: false
 toc: true
-weight: 802
+weight: 801
 ---
 
 ## 更新指南

docSite/content/zh-cn/docs/development/upgrading/490.md

@@ -4,7 +4,7 @@ description: 'FastGPT V4.9.0 更新说明'
 icon: 'upgrade'
 draft: false
 toc: true
-weight: 801
+weight: 800
 ---
 
 

docSite/content/zh-cn/docs/development/upgrading/491.md

@@ -0,0 +1,24 @@
+---
+title: 'V4.9.1'
+description: 'FastGPT V4.9.1 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 799
+---
+
+## 🚀 新增内容
+
+
+## ⚙️ 优化
+
+1. 知识库数据输入框交互
+2. 应用拉取绑定知识库数据交由后端处理。
+3. 增加依赖包安全版本检测，并升级部分依赖包。
+
+## 🐛 修复
+
+1. 最大响应 tokens 提示显示错误的问题。
+2. HTTP Node 中，字符串包含换行符时，会解析失败。
+3. 知识库问题优化中，未传递历史记录。
+4. 错误提示翻译缺失。

packages/global/common/string/textSplitter.ts

@@ -93,7 +93,7 @@ ${mdSplitString}
 
 /* 
   1. 自定义分隔符：不需要重叠，不需要小块合并
-  2. Markdown 标题：不需要重叠；标题嵌套共享，不需要小块合并
+  2. Markdown 标题：不需要重叠；标题嵌套共享，需要小块合并
   3. 特殊 markdown 语法：不需要重叠，需要小块合并
   4. 段落：尽可能保证它是一个完整的段落。
   5. 标点分割：重叠
@@ -227,7 +227,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
   }): string[] => {
     const isMarkdownStep = checkIsMarkdownSplit(step);
     const isCustomStep = checkIsCustomStep(step);
-    const forbidConcat = isMarkdownStep || isCustomStep; // forbid=true时候，lastText肯定为空
+    const forbidConcat = isCustomStep; // forbid=true时候，lastText肯定为空
 
     // oversize
     if (step >= stepReges.length) {

packages/service/common/vectorStore/pg/class.ts

@@ -38,6 +38,27 @@ export class PgVectorCtrl {
       await PgClient.query(
         `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${DatasetVectorTableName} USING btree(createtime);`
       );
+      // 10w rows
+      // await PgClient.query(`
+      //   ALTER TABLE modeldata SET (
+      //     autovacuum_vacuum_scale_factor = 0.1,
+      //     autovacuum_analyze_scale_factor = 0.05,
+      //     autovacuum_vacuum_threshold = 50,
+      //     autovacuum_analyze_threshold = 50,
+      //     autovacuum_vacuum_cost_delay = 20,
+      //     autovacuum_vacuum_cost_limit = 200
+      //   );`);
+
+      // 100w rows
+      // await PgClient.query(`
+      //   ALTER TABLE modeldata SET (
+      //   autovacuum_vacuum_scale_factor = 0.01,
+      //   autovacuum_analyze_scale_factor = 0.02,
+      //   autovacuum_vacuum_threshold = 1000,
+      //   autovacuum_analyze_threshold = 1000,
+      //   autovacuum_vacuum_cost_delay = 10,
+      //   autovacuum_vacuum_cost_limit = 2000
+      // );`)
 
       addLog.info('init pg successful');
     } catch (error) {

packages/service/core/app/utils.ts

@@ -4,20 +4,13 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
 
-export type ListByAppIdAndDatasetIdsBody = {
-  teamId: string;
-  datasetIdList: string[];
-};
-
-interface Dataset {
-  datasetId: string;
-  [key: string]: any;
-}
-
 export async function listAppDatasetDataByTeamIdAndDatasetIds({
   teamId,
   datasetIdList
-}: ListByAppIdAndDatasetIdsBody) {
+}: {
+  teamId?: string;
+  datasetIdList: string[];
+}) {
   const myDatasets = await MongoDataset.find({
     teamId,
     _id: { $in: datasetIdList }
@@ -31,93 +24,102 @@ export async function listAppDatasetDataByTeamIdAndDatasetIds({
   }));
 }
 
-export async function rewriteAppWorkflowToDetail(nodes: StoreNodeItemType[], teamId: string) {
+export async function rewriteAppWorkflowToDetail({
+  nodes,
+  teamId,
+  isRoot
+}: {
+  nodes: StoreNodeItemType[];
+  teamId: string;
+  isRoot: boolean;
+}) {
   const datasetIdSet = new Set<string>();
 
+  // Get all dataset ids from nodes
   nodes.forEach((node) => {
     if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
 
     const input = node.inputs.find((item) => item.key === NodeInputKeyEnum.datasetSelectList);
     if (!input) return;
 
     const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string };
+    if (!rawValue) return;
 
     const datasetIds = Array.isArray(rawValue)
-      ? rawValue
-          .map((v) => v?.datasetId)
-          .filter((id): id is string => !!id && typeof id === 'string')
-      : rawValue?.datasetId
+      ? rawValue.map((v) => v?.datasetId).filter((id) => !!id && typeof id === 'string')
+      : rawValue.datasetId
         ? [String(rawValue.datasetId)]
         : [];
 
-    if (datasetIds.length === 0) return;
-
     datasetIds.forEach((id) => datasetIdSet.add(id));
   });
 
   if (datasetIdSet.size === 0) return;
 
-  const uniqueDatasetIds = Array.from(datasetIdSet);
+  // Load dataset list
   const datasetList = await listAppDatasetDataByTeamIdAndDatasetIds({
-    teamId,
-    datasetIdList: uniqueDatasetIds
+    teamId: isRoot ? undefined : teamId,
+    datasetIdList: Array.from(datasetIdSet)
   });
+  const datasetMap = new Map(datasetList.map((ds) => [String(ds.datasetId), ds]));
 
-  const datasetMap = new Map(
-    datasetList.map((ds) => [
-      String(ds.datasetId),
-      {
-        ...ds,
-        vectorModel: getEmbeddingModel(ds.vectorModel.model)
-      }
-    ])
-  );
-
+  // Rewrite dataset ids, add dataset info to nodes
   nodes.forEach((node) => {
     if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
 
-    const input = node.inputs.find((item) => item.key === NodeInputKeyEnum.datasetSelectList);
-    if (!input) return;
-
-    const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string };
-
-    const datasetIds = Array.isArray(rawValue)
-      ? rawValue
-          .map((v) => v?.datasetId)
-          .filter((id): id is string => !!id && typeof id === 'string')
-      : rawValue?.datasetId
-        ? [String(rawValue.datasetId)]
-        : [];
-
-    if (datasetIds.length === 0) return;
-
-    input.value = datasetIds
-      .map((id) => {
-        const data = datasetMap.get(String(id));
-        return data
-          ? {
-              datasetId: data.datasetId,
-              avatar: data.avatar,
-              name: data.name,
-              vectorModel: data.vectorModel
-            }
-          : undefined;
-      })
-      .filter((item): item is NonNullable<typeof item> => !!item);
+    node.inputs.forEach((item) => {
+      if (item.key !== NodeInputKeyEnum.datasetSelectList) return;
+
+      const val = item.value as undefined | { datasetId: string }[] | { datasetId: string };
+
+      if (Array.isArray(val)) {
+        item.value = val.map((v) => {
+          const data = datasetMap.get(String(v.datasetId))!;
+          return {
+            datasetId: data.datasetId,
+            avatar: data.avatar,
+            name: data.name,
+            vectorModel: data.vectorModel
+          };
+        });
+      } else if (typeof val === 'object' && val !== null) {
+        const data = datasetMap.get(String(val.datasetId))!;
+        item.value = {
+          datasetId: data.datasetId,
+          avatar: data.avatar,
+          name: data.name,
+          vectorModel: data.vectorModel
+        };
+      }
+    });
   });
+
+  return nodes;
 }
 
 export async function rewriteAppWorkflowToSimple(formatNodes: StoreNodeItemType[]) {
   formatNodes.forEach((node) => {
     if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
 
-    const datasetsInput = node.inputs.find(
-      (input) => input.key === NodeInputKeyEnum.datasetSelectList
-    );
-    if (datasetsInput?.value) {
-      datasetsInput.value = datasetsInput.value.map((dataset: Dataset) => ({
-        datasetId: dataset.datasetId
-      }));
-    }
+    node.inputs.forEach((input) => {
+      if (input.key === NodeInputKeyEnum.datasetSelectList) {
+        const val = input.value as undefined | { datasetId: string }[] | { datasetId: string };
+        if (!val) {
+          input.value = [];
+        } else if (Array.isArray(val)) {
+          input.value = val
+            .map((dataset: { datasetId: string }) => ({
+              datasetId: dataset.datasetId
+            }))
+            .filter((item) => !!item.datasetId);
+        } else if (typeof val === 'object' && val !== null) {
+          input.value = [
+            {
+              datasetId: val.datasetId
+            }
+          ];
+        }
+      }
+    });
   });
 }

packages/web/components/common/Icon/button.tsx

@@ -34,6 +34,7 @@ const MyIconButton = ({
           color: hoverColor
         }}
         onClick={onClick}
+        sx={{ userSelect: 'none' }}
         {...props}
       >
         <MyIcon name={icon as any} w={size} />

packages/web/components/common/Icon/constants.ts

@@ -373,9 +373,13 @@ export const iconPaths = {
   'modal/teamPlans': () => import('./icons/modal/teamPlans.svg'),
   'model/BAAI': () => import('./icons/model/BAAI.svg'),
   'model/alicloud': () => import('./icons/model/alicloud.svg'),
+  'model/aws': () => import('./icons/model/aws.svg'),
+  'model/azure': () => import('./icons/model/azure.svg'),
   'model/baichuan': () => import('./icons/model/baichuan.svg'),
   'model/chatglm': () => import('./icons/model/chatglm.svg'),
   'model/claude': () => import('./icons/model/claude.svg'),
+  'model/cloudflare': () => import('./icons/model/cloudflare.svg'),
+  'model/cohere': () => import('./icons/model/cohere.svg'),
   'model/deepseek': () => import('./icons/model/deepseek.svg'),
   'model/doubao': () => import('./icons/model/doubao.svg'),
   'model/ernie': () => import('./icons/model/ernie.svg'),