feat: Sync collection (#3368)

c121914yu · web-flow · commit d5752ddbaa03 · 2024-12-11T15:03:41.000+08:00
* feat: sync collection

* feat: sync collection

* perf: website selector

* update doc
diff --git a/docSite/content/zh-cn/docs/development/upgrading/43.md b/docSite/content/zh-cn/docs/development/upgrading/43.md
@@ -1,5 +1,5 @@
 ---
-title: '升级到 V4.3(需要初始化)'
+title: '升级到 V4.3(包含升级脚本)'
 description: 'FastGPT 从旧版本升级到 V4.3 操作指南'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/44.md b/docSite/content/zh-cn/docs/development/upgrading/44.md
@@ -1,5 +1,5 @@
 ---
-title: '升级到 V4.4(需要初始化)'
+title: '升级到 V4.4(包含升级脚本)'
 description: 'FastGPT 从旧版本升级到 V4.4 操作指南'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/441.md b/docSite/content/zh-cn/docs/development/upgrading/441.md
@@ -1,5 +1,5 @@
 ---
-title: '升级到 V4.4.1(需要初始化)'
+title: '升级到 V4.4.1(包含升级脚本)'
 description: 'FastGPT 从旧版本升级到 V4.4.1 操作指南'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/442.md b/docSite/content/zh-cn/docs/development/upgrading/442.md
@@ -1,5 +1,5 @@
 ---
-title: '升级到 V4.4.2(需要初始化)'
+title: '升级到 V4.4.2(包含升级脚本)'
 description: 'FastGPT 从旧版本升级到 V4.4.2 操作指南'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/445.md b/docSite/content/zh-cn/docs/development/upgrading/445.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.4.5(需要初始化)'
+title: 'V4.4.5(包含升级脚本)'
 description: 'FastGPT V4.4.5 更新'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/46.md b/docSite/content/zh-cn/docs/development/upgrading/46.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.6(需要初始化)'
+title: 'V4.6(包含升级脚本)'
 description: 'FastGPT V4.6 更新'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/462.md b/docSite/content/zh-cn/docs/development/upgrading/462.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.6.2(需要初始化)'
+title: 'V4.6.2(包含升级脚本)'
 description: 'FastGPT V4.6.2'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/463.md b/docSite/content/zh-cn/docs/development/upgrading/463.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.6.3(需要初始化)'
+title: 'V4.6.3(包含升级脚本)'
 description: 'FastGPT V4.6.3'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/464.md b/docSite/content/zh-cn/docs/development/upgrading/464.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.6.4(需要初始化)'
+title: 'V4.6.4(包含升级脚本)'
 description: 'FastGPT V4.6.4'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/469.md b/docSite/content/zh-cn/docs/development/upgrading/469.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.6.9(需要初始化)'
+title: 'V4.6.9(包含升级脚本)'
 description: 'FastGPT V4.6.9更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/471.md b/docSite/content/zh-cn/docs/development/upgrading/471.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.7.1(需要初始化)'
+title: 'V4.7.1(包含升级脚本)'
 description: 'FastGPT V4.7.1 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/481.md b/docSite/content/zh-cn/docs/development/upgrading/481.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.1(需要初始化)'
+title: 'V4.8.1(包含升级脚本)'
 description: 'FastGPT V4.8.1 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/4810.md b/docSite/content/zh-cn/docs/development/upgrading/4810.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.10(需要初始化)'
+title: 'V4.8.10(包含升级脚本)'
 description: 'FastGPT V4.8.10 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/4812.md b/docSite/content/zh-cn/docs/development/upgrading/4812.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.12(需要初始化)'
+title: 'V4.8.12(包含升级脚本)'
 description: 'FastGPT V4.8.12 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/4815.md b/docSite/content/zh-cn/docs/development/upgrading/4815.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.15(初始化)'
+title: 'V4.8.15(包含升级脚本)'
 description: 'FastGPT V4.8.15 更新说明'
 icon: 'upgrade'
 draft: false
@@ -23,12 +23,12 @@ weight: 809
 
 ## 升级指南
 
-- 更新 FastGPT 镜像 tag: v4.8.15
+- 更新 FastGPT 镜像 tag: v4.8.15-fix-emb-page
 - 更新 FastGPT 商业版镜像 tag: v4.8.15 （fastgpt-pro镜像）
 - Sandbox 镜像，可以不更新
 
 
-## 运行初始化脚本
+## 运行升级脚本
 
 从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
 
diff --git a/docSite/content/zh-cn/docs/development/upgrading/4816.md b/docSite/content/zh-cn/docs/development/upgrading/4816.md
@@ -0,0 +1,15 @@
+---
+title: 'V4.8.16(进行中)'
+description: 'FastGPT V4.8.16 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 808
+---
+
+
+## 完整更新内容
+
+1. 
+2. 新增 - 商业版支持 API 知识库和链接集合定时同步。
+3. 修复 - 站点同步知识库，链接同步时未使用选择器。
diff --git a/docSite/content/zh-cn/docs/development/upgrading/484.md b/docSite/content/zh-cn/docs/development/upgrading/484.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.4(需要初始化)'
+title: 'V4.8.4(包含升级脚本)'
 description: 'FastGPT V4.8.4 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/485.md b/docSite/content/zh-cn/docs/development/upgrading/485.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.5(需要初始化)'
+title: 'V4.8.5(包含升级脚本)'
 description: 'FastGPT V4.8.5 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/486.md b/docSite/content/zh-cn/docs/development/upgrading/486.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.6(需要初始化)'
+title: 'V4.8.6(包含升级脚本)'
 description: 'FastGPT V4.8.6 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/488.md b/docSite/content/zh-cn/docs/development/upgrading/488.md
@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.8(需要初始化)'
+title: 'V4.8.8(包含升级脚本)'
 description: 'FastGPT V4.8.8 更新说明'
 icon: 'upgrade'
 draft: false
diff --git a/docSite/content/zh-cn/docs/development/upgrading/intro.md b/docSite/content/zh-cn/docs/development/upgrading/intro.md
@@ -55,14 +55,14 @@ docker-compose up -d
 
 ## 执行升级初始化脚本
 
-镜像更新完后，可以查看文档中的`版本介绍`，通常需要执行升级脚本的版本都会标明`需要初始化`，打开对应的文档，参考说明执行初始化脚本即可，大部分时候都是需要发送一个`POST`请求。
+镜像更新完后，可以查看文档中的`版本介绍`，通常需要执行升级脚本的版本都会标明`包含升级脚本`，打开对应的文档，参考说明执行**升级脚本**即可，大部分时候都是需要发送一个`POST`请求。
 
 
 ## QA
 
-### 为什么需要初始化
+### 为什么需要执行升级脚本
 
-数据表出现大幅度变更，无法通过设置默认值，或复杂度较高时，会通过初始化来更新部分数据表字段。
+数据表出现大幅度变更，无法通过设置默认值，或复杂度较高时，会通过升级脚本来更新部分数据表字段。
 严格按初始化步骤进行操作，不会造成旧数据丢失。但在初始化过程中，如果数据量大，需要初始化的时间较长，这段时间可能会造成服务无法正常使用。
 
 ### {{host}} 是什么
diff --git a/files/docker/docker-compose-milvus.yml b/files/docker/docker-compose-milvus.yml
@@ -121,8 +121,8 @@ services:
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.8.15 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15 # 阿里云
+    image: ghcr.io/labring/fastgpt:v4.8.15-fix-emb-page # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15-fix-emb-page # 阿里云
     ports:
       - 3000:3000
     networks:
diff --git a/files/docker/docker-compose-pgvector.yml b/files/docker/docker-compose-pgvector.yml
@@ -79,8 +79,8 @@ services:
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.8.15 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15 # 阿里云
+    image: ghcr.io/labring/fastgpt:v4.8.15-fix-emb-page # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15-fix-emb-page # 阿里云
     ports:
       - 3000:3000
     networks:
diff --git a/files/docker/docker-compose-zilliz.yml b/files/docker/docker-compose-zilliz.yml
@@ -60,8 +60,8 @@ services:
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.8.15 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15 # 阿里云
+    image: ghcr.io/labring/fastgpt:v4.8.15-fix-emb-page # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.15-fix-emb-page # 阿里云
     ports:
       - 3000:3000
     networks:
diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts
@@ -17,6 +17,9 @@ export type DatasetUpdateBody = {
   externalReadUrl?: DatasetSchemaType['externalReadUrl'];
   defaultPermission?: DatasetSchemaType['defaultPermission'];
   apiServer?: DatasetSchemaType['apiServer'];
+
+  // sync schedule
+  autoSync?: boolean;
 };
 
 /* ================= collection ===================== */
@@ -47,6 +50,8 @@ export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType &
   tags?: string[];
 
   createTime?: Date;
+  updateTime?: Date;
+  nextSyncTime?: Date;
 };
 
 export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts
@@ -82,14 +82,18 @@ export const DatasetCollectionTypeMap = {
 
 export enum DatasetCollectionSyncResultEnum {
   sameRaw = 'sameRaw',
-  success = 'success'
+  success = 'success',
+  failed = 'failed'
 }
 export const DatasetCollectionSyncResultMap = {
   [DatasetCollectionSyncResultEnum.sameRaw]: {
     label: i18nT('common:core.dataset.collection.sync.result.sameRaw')
   },
   [DatasetCollectionSyncResultEnum.success]: {
     label: i18nT('common:core.dataset.collection.sync.result.success')
+  },
+  [DatasetCollectionSyncResultEnum.failed]: {
+    label: i18nT('dataset:sync_collection_failed')
   }
 };
 
diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts
@@ -34,8 +34,7 @@ export type DatasetSchemaType = {
   inheritPermission: boolean;
   apiServer?: APIFileServer;
 
-  syncSchedule?: { cronString: string; timezone: string };
-  syncNextTime?: Date;
+  autoSync?: boolean;
 
   // abandon
   externalReadUrl?: string;
@@ -65,11 +64,13 @@ export type DatasetCollectionSchemaType = {
   fileId?: string; // local file id
   rawLink?: string; // link url
   externalFileId?: string; //external file id
+  apiFileId?: string; // api file id
+  externalFileUrl?: string; // external import url
+
+  nextSyncTime?: Date;
 
   rawTextLength?: number;
   hashRawText?: string;
-  externalFileUrl?: string; // external import url
-  apiFileId?: string; // api file id
   metadata?: {
     webPageSelector?: string;
     relatedImgId?: string; // The id of the associated image collections
diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts
@@ -1,4 +1,7 @@
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import {
+  DatasetCollectionTypeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constants';
 import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
 import { MongoDatasetCollection } from './schema';
 import {
@@ -24,6 +27,7 @@ import { getLLMModel, getVectorModel } from '../../ai/model';
 import { pushDataListToTrainingQueue } from '../training/controller';
 import { MongoImage } from '../../../common/file/image/schema';
 import { hashStr } from '@fastgpt/global/common/string/tools';
+import { addDays } from 'date-fns';
 
 export const createCollectionAndInsertData = async ({
   dataset,
@@ -72,6 +76,17 @@ export const createCollectionAndInsertData = async ({
 
       hashRawText: hashStr(rawText),
       rawTextLength: rawText.length,
+      nextSyncTime: (() => {
+        if (!dataset.autoSync) return undefined;
+        if (
+          [DatasetCollectionTypeEnum.link, DatasetCollectionTypeEnum.apiFile].includes(
+            createCollectionParams.type
+          )
+        ) {
+          return addDays(new Date(), 1);
+        }
+        return undefined;
+      })(),
       session
     });
 
@@ -155,18 +170,19 @@ export async function createOneCollection({
 
   fileId,
   rawLink,
-
   externalFileId,
   externalFileUrl,
-
   apiFileId,
 
   hashRawText,
   rawTextLength,
   metadata = {},
   session,
   tags,
-  createTime
+
+  createTime,
+  updateTime,
+  nextSyncTime
 }: CreateOneCollectionParams) {
   // Create collection tags
   const collectionTags = await createOrGetCollectionTags({ tags, teamId, datasetId, session });
@@ -197,7 +213,10 @@ export async function createOneCollection({
         rawTextLength,
         hashRawText,
         tags: collectionTags,
-        createTime
+
+        createTime,
+        updateTime,
+        nextSyncTime
       }
     ],
     { session }
diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts
diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts
diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts
diff --git a/packages/service/core/dataset/training/controller.ts b/packages/service/core/dataset/training/controller.ts
diff --git a/packages/web/components/common/MySelect/CronSelector.tsx b/packages/web/components/common/MySelect/CronSelector.tsx
diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json
diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json
diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json
diff --git a/projects/app/src/components/Select/AIModelSelector.tsx b/projects/app/src/components/Select/AIModelSelector.tsx
diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts
diff --git a/projects/app/src/pages/dataset/detail/components/Info/index.tsx b/projects/app/src/pages/dataset/detail/components/Info/index.tsx
diff --git a/projects/app/src/pages/dataset/detail/components/MetaDataCard.tsx b/projects/app/src/pages/dataset/detail/components/MetaDataCard.tsx
