feat: api dataset support pdf parse;fix: chunk reader auth (#4117)

* feat: api dataset support pdf parse

* fix: chunk reader auth

Author: c121914yu

docSite/content/zh-cn/docs/development/upgrading/491.md

@@ -11,6 +11,7 @@ weight: 799
 
 1. 商业版支持单团队模式，更好的管理内部成员。
 2. 知识库分块阅读器。
+3. API 知识库支持 PDF 增强解析。
 
 ## ⚙️ 优化
 

packages/service/core/dataset/apiDataset/api.ts

@@ -111,11 +111,13 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
   const getFileContent = async ({
     teamId,
     tmbId,
-    apiFileId
+    apiFileId,
+    customPdfParse
   }: {
     teamId: string;
     tmbId: string;
     apiFileId: string;
+    customPdfParse?: boolean;
   }) => {
     const data = await request<APIFileContentResponse>(
       `/v1/file/content`,
@@ -133,7 +135,8 @@ export const useApiDatasetRequest = ({ apiServer }: { apiServer: APIFileServer }
         teamId,
         tmbId,
         url: previewUrl,
-        relatedId: apiFileId
+        relatedId: apiFileId,
+        customPdfParse
       });
       return rawText;
     }

packages/service/core/dataset/read.ts

@@ -127,17 +127,24 @@ export const readApiServerFileContent = async ({
   yuqueServer,
   apiFileId,
   teamId,
-  tmbId
+  tmbId,
+  customPdfParse
 }: {
   apiServer?: APIFileServer;
   feishuServer?: FeishuServer;
   yuqueServer?: YuqueServer;
   apiFileId: string;
   teamId: string;
   tmbId: string;
+  customPdfParse?: boolean;
 }) => {
   if (apiServer) {
-    return useApiDatasetRequest({ apiServer }).getFileContent({ teamId, tmbId, apiFileId });
+    return useApiDatasetRequest({ apiServer }).getFileContent({
+      teamId,
+      tmbId,
+      apiFileId,
+      customPdfParse
+    });
   }
 
   if (feishuServer || yuqueServer) {

projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteList.tsx

@@ -11,19 +11,17 @@ import { useChatStore } from '@/web/core/chat/context/useChatStore';
 import { getQuoteDataList } from '@/web/core/chat/api';
 
 const QuoteList = React.memo(function QuoteList({
-  chatItemId,
-  rawSearch = [],
-  chatTime
+  chatItemDataId = '',
+  rawSearch = []
 }: {
-  chatItemId?: string;
+  chatItemDataId?: string;
   rawSearch: SearchDataResponseItemType[];
-  chatTime: Date;
 }) {
   const theme = useTheme();
   const { chatId, appId, outLinkAuthData } = useChatStore();
 
   const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({
-    chatItemId,
+    chatItemDataId,
     appId: v.appId,
     chatId: v.chatId,
     ...(v.outLinkAuthData || {})
@@ -34,13 +32,12 @@ const QuoteList = React.memo(function QuoteList({
     (v) => v.showRouteToDatasetDetail
   );
 
-  const { data } = useRequest2(
+  const { data: quoteList } = useRequest2(
     async () =>
       await getQuoteDataList({
         datasetDataIdList: rawSearch.map((item) => item.id),
-        chatTime,
         collectionIdList: [...new Set(rawSearch.map((item) => item.collectionId))],
-        chatItemId: chatItemId || '',
+        chatItemDataId,
         appId,
         chatId,
         ...outLinkAuthData
@@ -53,7 +50,7 @@ const QuoteList = React.memo(function QuoteList({
   const formatedDataList = useMemo(() => {
     return rawSearch
       .map((item) => {
-        const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
+        const currentFilterItem = quoteList?.find((res) => res._id === item.id);
 
         return {
           ...item,
@@ -66,7 +63,7 @@ const QuoteList = React.memo(function QuoteList({
         const bScore = formatScore(b.score);
         return (bScore.primaryScore?.value || 0) - (aScore.primaryScore?.value || 0);
       });
-  }, [data?.quoteList, rawSearch]);
+  }, [quoteList, rawSearch]);
 
   return (
     <>

projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ResponseTags.tsx

@@ -42,6 +42,8 @@ const ResponseTags = ({
   const [quoteFolded, setQuoteFolded] = useState<boolean>(true);
 
   const chatType = useContextSelector(ChatBoxContext, (v) => v.chatType);
+  const appId = useContextSelector(ChatBoxContext, (v) => v.appId);
+  const chatId = useContextSelector(ChatBoxContext, (v) => v.chatId);
 
   const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData);
 
@@ -156,17 +158,15 @@ const ResponseTags = ({
                       e.stopPropagation();
 
                       setQuoteData({
-                        chatTime,
                         rawSearch: quoteList,
                         metadata: {
+                          appId,
+                          chatId,
+                          chatItemDataId: dataId,
                           collectionId: item.collectionId,
-                          collectionIdList: [
-                            ...new Set(quoteList.map((item) => item.collectionId))
-                          ],
                           sourceId: item.sourceId || '',
                           sourceName: item.sourceName,
-                          datasetId: item.datasetId,
-                          chatItemId: historyItem.dataId
+                          datasetId: item.datasetId
                         }
                       });
                     }}
@@ -225,15 +225,12 @@ const ResponseTags = ({
                   e.stopPropagation();
 
                   setQuoteData({
-                    chatTime,
                     rawSearch: quoteList,
                     metadata: {
-                      collectionId: '',
-                      collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))],
-                      chatItemId: historyItem.dataId,
-                      sourceId: '',
-                      sourceName: '',
-                      datasetId: ''
+                      appId,
+                      chatId,
+                      chatItemDataId: dataId,
+                      collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))]
                     }
                   });
                 }}

projects/app/src/components/core/chat/components/WholeResponseModal.tsx

@@ -265,13 +265,7 @@ export const WholeResponseContent = ({
         {activeModule.quoteList && activeModule.quoteList.length > 0 && (
           <Row
             label={t('common:core.chat.response.module quoteList')}
-            rawDom={
-              <QuoteList
-                chatItemId={dataId}
-                chatTime={chatTime || new Date()}
-                rawSearch={activeModule.quoteList}
-              />
-            }
+            rawDom={<QuoteList chatItemDataId={dataId} rawSearch={activeModule.quoteList} />}
           />
         )}
       </>

projects/app/src/components/core/dataset/QuoteItem.tsx

@@ -243,7 +243,7 @@ const QuoteItem = ({
               color={'primary.500'}
               href={`/dataset/detail?datasetId=${quoteItem.datasetId}&currentTab=dataCard&collectionId=${quoteItem.collectionId}`}
             >
-              {t('common:core.dataset.Go Dataset')}
+              {t('chat:to_dataset')}
               <MyIcon name={'common/rightArrowLight'} w={'10px'} />
             </Link>
           )}

projects/app/src/components/core/dataset/RawSourceBox.tsx

@@ -23,7 +23,7 @@ const RawSourceBox = ({
   collectionId,
   appId,
   chatId,
-  chatItemId,
+  chatItemDataId,
   shareId,
   outLinkUid,
   teamId,
@@ -40,7 +40,7 @@ const RawSourceBox = ({
     collectionId,
     appId,
     chatId,
-    chatItemId,
+    chatItemDataId,
     shareId,
     outLinkUid,
     teamId,

projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx

@@ -184,7 +184,6 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => {
               borderRadius={'md'}
             >
               <ChatQuoteList
-                chatTime={quoteData.chatTime}
                 rawSearch={quoteData.rawSearch}
                 metadata={quoteData.metadata}
                 onClose={() => setQuoteData(undefined)}

projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx

@@ -88,7 +88,6 @@ const ChatTest = ({ appForm, setRenderEdit }: Props) => {
       {quoteData && (
         <Box flex={'1 0 0'} w={0} maxW={'560px'} {...cardStyles} boxShadow={'3'}>
           <ChatQuoteList
-            chatTime={quoteData.chatTime}
             rawSearch={quoteData.rawSearch}
             metadata={quoteData.metadata}
             onClose={() => setQuoteData(undefined)}