fix: mongodb file oversize (#4594)

Author: c121914yu

docSite/content/zh-cn/docs/development/upgrading/497.md

@@ -16,4 +16,5 @@ weight: 793
 
 ## 🐛 修复
 
+1. 文件上传分块大小限制，避免超出 MongoDB 限制。
 

packages/service/common/file/gridfs/controller.ts

@@ -65,9 +65,10 @@ export async function uploadFile({
   const bucket = getGridBucket(bucketName);
 
   const fileSize = stats.size;
+  // 单块大小：尽可能大，但不超过 14MB，不小于512KB
   const chunkSizeBytes = (() => {
-    // 计算理想块大小：文件大小 ÷ 目标块数(10)
-    const idealChunkSize = Math.ceil(fileSize / 10);
+    // 计算理想块大小：文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
+    const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
 
     // 确保块大小至少为512KB
     const minChunkSize = 512 * 1024; // 512KB

projects/app/src/pageComponents/dataset/detail/Import/components/FileSelector.tsx

@@ -16,6 +16,7 @@ import { ImportSourceItemType } from '@/web/core/dataset/type';
 import { useI18n } from '@/web/context/I18n';
 import { useContextSelector } from 'use-context-selector';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 
 export type SelectFileItemType = {
   fileId: string;
@@ -71,39 +72,53 @@ const FileSelector = ({
       {
         await Promise.all(
           files.map(async ({ fileId, file }) => {
-            const { fileId: uploadFileId } = await uploadFile2DB({
-              file,
-              bucketName: BucketNameEnum.dataset,
-              data: {
-                datasetId
-              },
-              percentListen: (e) => {
-                setSelectFiles((state) =>
-                  state.map((item) =>
-                    item.id === fileId
-                      ? {
-                          ...item,
-                          uploadedFileRate: item.uploadedFileRate
-                            ? Math.max(e, item.uploadedFileRate)
-                            : e
-                        }
-                      : item
-                  )
-                );
-              }
-            });
-            setSelectFiles((state) =>
-              state.map((item) =>
-                item.id === fileId
-                  ? {
-                      ...item,
-                      dbFileId: uploadFileId,
-                      isUploading: false,
-                      uploadedFileRate: 100
-                    }
-                  : item
-              )
-            );
+            try {
+              const { fileId: uploadFileId } = await uploadFile2DB({
+                file,
+                bucketName: BucketNameEnum.dataset,
+                data: {
+                  datasetId
+                },
+                percentListen: (e) => {
+                  setSelectFiles((state) =>
+                    state.map((item) =>
+                      item.id === fileId
+                        ? {
+                            ...item,
+                            uploadedFileRate: item.uploadedFileRate
+                              ? Math.max(e, item.uploadedFileRate)
+                              : e
+                          }
+                        : item
+                    )
+                  );
+                }
+              });
+              setSelectFiles((state) =>
+                state.map((item) =>
+                  item.id === fileId
+                    ? {
+                        ...item,
+                        dbFileId: uploadFileId,
+                        isUploading: false,
+                        uploadedFileRate: 100
+                      }
+                    : item
+                )
+              );
+            } catch (error) {
+              setSelectFiles((state) =>
+                state.map((item) =>
+                  item.id === fileId
+                    ? {
+                        ...item,
+                        isUploading: false,
+                        errorMsg: getErrText(error)
+                      }
+                    : item
+                )
+              );
+            }
           })
         );
       }

projects/app/src/pageComponents/dataset/detail/Import/components/RenderFiles.tsx

@@ -9,12 +9,16 @@ import {
   Td,
   Tbody,
   Progress,
-  IconButton
+  IconButton,
+  Box
 } from '@chakra-ui/react';
 import { ImportSourceItemType } from '@/web/core/dataset/type.d';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useTranslation } from 'next-i18next';
 import { useI18n } from '@/web/context/I18n';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import MyTag from '@fastgpt/web/components/common/Tag/index';
+import { QuestionOutlineIcon } from '@chakra-ui/icons';
 
 export const RenderUploadFiles = ({
   files,
@@ -56,22 +60,31 @@ export const RenderUploadFiles = ({
                   </Flex>
                 </Td>
                 <Td>
-                  <Flex alignItems={'center'} fontSize={'xs'}>
-                    <Progress
-                      value={item.uploadedFileRate}
-                      h={'6px'}
-                      w={'100%'}
-                      maxW={'210px'}
-                      size="sm"
-                      borderRadius={'20px'}
-                      colorScheme={(item.uploadedFileRate || 0) >= 100 ? 'green' : 'blue'}
-                      bg="myGray.200"
-                      hasStripe
-                      isAnimated
-                      mr={2}
-                    />
-                    {`${item.uploadedFileRate}%`}
-                  </Flex>
+                  {item.errorMsg ? (
+                    <MyTooltip label={item.errorMsg}>
+                      <MyTag colorSchema={'red'}>
+                        <Box mr={1}>{t('common:common.Error')}</Box>
+                        <MyIcon name={'help'} w={'0.9rem'} color={'red.500'} />
+                      </MyTag>
+                    </MyTooltip>
+                  ) : (
+                    <Flex alignItems={'center'} fontSize={'xs'}>
+                      <Progress
+                        value={item.uploadedFileRate}
+                        h={'6px'}
+                        w={'100%'}
+                        maxW={'210px'}
+                        size="sm"
+                        borderRadius={'20px'}
+                        colorScheme={(item.uploadedFileRate || 0) >= 100 ? 'green' : 'blue'}
+                        bg="myGray.200"
+                        hasStripe
+                        isAnimated
+                        mr={2}
+                      />
+                      {`${item.uploadedFileRate}%`}
+                    </Flex>
+                  )}
                 </Td>
                 <Td>{item.sourceSize}</Td>
                 <Td>