fix: bugs (#739)

vesoft-inc · Jan 16, 2024 · e7e4ab3 · e7e4ab3
1 parent 332c16e
commit e7e4ab3
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 85 deletions.
diff --git a/app/pages/Import/AIImport/Create.tsx b/app/pages/Import/AIImport/Create.tsx
@@ -44,17 +44,21 @@ const Create = observer((props: { visible: boolean; onCancel: () => void }) => {
     (async () => {
       if (file && space) {
         const types = {
-          csv: 0.9,
-          json: 0.6,
+          csv: 1,
+          txt: 1,
+          json: 0.8,
           pdf: 0.05,
         };
         const subfix = file.name.split('.').pop();
-        const type = types[subfix] || 0.5;
-        const size = file.size;
+        const type = types[subfix] || 1;
+        const size = file.size * type;
         const schema = await llm.getSpaceSchema(space);
-        const schemaBytesLength = getByteLength(schema);
+        const schemaBytesLength = getByteLength(schema) + 500; //prompt length about 500
+        const splitNums = Math.ceil(size / llm.config.maxContextLength);
+        const splitBytesLength = Math.ceil(schemaBytesLength * splitNums);
+        const totalBytesLength = splitBytesLength + file.size;
         // full connection
-        const tokensNum = ((((schemaBytesLength * size) / 2000) * llm.config.maxContextLength) / 2000) * type;
+        const tokensNum = totalBytesLength / 4; //about 4 bytes per token
         setTokens(tokensNum);
       }
     })();
@@ -89,20 +93,20 @@ const Create = observer((props: { visible: boolean; onCancel: () => void }) => {
             setStep(0);
           }}
         >
-          <Icon type={step === 0 ? "icon-vesoft-numeric-1-circle":"icon-vesoft-check-circle-filled"} />
+          <Icon type={step === 0 ? 'icon-vesoft-numeric-1-circle' : 'icon-vesoft-check-circle-filled'} />
           {intl.get('llm.setup')}
         </div>
         <span />
-        <div style={{color:step===0?'#888':'#0D8BFF'}} >
+        <div style={{ color: step === 0 ? '#888' : '#0D8BFF' }}>
           <Icon type="icon-vesoft-numeric-2-circle" />
           {intl.get('llm.confirm')}
         </div>
       </div>
-      {tokens !== 0 && type!=="filePath" && (
-        <Tooltip title={intl.get("llm.tokenTip")} placement="bottom">
+      {tokens !== 0 && type !== 'filePath' && (
+        <Tooltip title={intl.get('llm.tokenTip')} placement="bottom">
           <div className={styles.tokenNum}>
-          <span style={{ fontSize: 14, transform: 'translate(0px,1px)' }}>🅣</span> prompt token: ~
-          {Math.ceil(tokens / 10000)}w
+            <span style={{ fontSize: 14, transform: 'translate(0px,1px)' }}>🅣</span> prompt token: ~
+            {Math.ceil(tokens / 10000)}w
           </div>
         </Tooltip>
       )}

diff --git a/app/pages/Import/DatasourceList/LocalFileList/PreviewFileModal/index.tsx b/app/pages/Import/DatasourceList/LocalFileList/PreviewFileModal/index.tsx
@@ -9,11 +9,13 @@ import { useBatchState } from '@app/utils';
 import { useStore } from '@app/stores';
 import styles from './index.module.less';
 
-type IProps = PropsWithChildren<{ file: StudioFile}>
+type IProps = PropsWithChildren<{ file: StudioFile }>;
 
 const PreviewFileModal = (props: IProps) => {
   const { children, file } = props;
-  const { files: { updateFileConfig, getFiles } } = useStore();
+  const {
+    files: { updateFileConfig, getFiles },
+  } = useStore();
   const [form] = Form.useForm();
   const { state, setState } = useBatchState({
     visible: false,
@@ -33,32 +35,36 @@ const PreviewFileModal = (props: IProps) => {
     parseColumn(withHeader);
   }, [data]);
   const parseColumn = (withHeader: boolean) => {
-    const columns = data[0]?.map((header, index) => {
-      const textIndex = index;
-      const title = withHeader ? header : `Column ${textIndex}`;
-      return {
-        title,
-        dataIndex: index,
-        render: value => <span className={styles.limitWidth}>{value}</span>,
-      };
-    }) || [];
+    const columns =
+      data[0]?.map((header, index) => {
+        const textIndex = index;
+        const title = withHeader ? header : `Column ${textIndex}`;
+        return {
+          title,
+          dataIndex: index,
+          render: (value) => <span className={styles.limitWidth}>{value}</span>,
+        };
+      }) || [];
     setState({ columns });
   };
-  const readFile = useCallback((delimiter?: string) => {
-    setState({ parseLoading: true });
-    let data = [];
-    readString(sample, { 
-      delimiter: delimiter || file.delimiter, 
-      worker: true, 
-      skipEmptyLines: true,
-      step: (row) => {
-        data = [...data, row.data];
-      },
-      complete: () => {
-        setState({ parseLoading: false, data });
-      } 
-    });
-  }, [sample, delimiter]);
+  const readFile = useCallback(
+    (delimiter?: string) => {
+      setState({ parseLoading: true });
+      let data = [];
+      readString(sample, {
+        delimiter: delimiter || file.delimiter,
+        worker: true,
+        skipEmptyLines: true,
+        step: (row) => {
+          data = [...data, row.data];
+        },
+        complete: () => {
+          setState({ parseLoading: false, data });
+        },
+      });
+    },
+    [sample, delimiter],
+  );
   const handleConfirm = async () => {
     setState({ uploading: true });
     const { withHeader, delimiter } = form.getFieldsValue();
@@ -87,22 +93,28 @@ const PreviewFileModal = (props: IProps) => {
         footer={false}
       >
         <div className={styles.container}>
-          <Form form={form} layout="horizontal" initialValues={{
-            withHeader,
-            delimiter,
-          }}>
-            <Row className={styles.configOperation}>
-              <Col span={3}>
-                <Form.Item name="withHeader" valuePropName="checked">
-                  <Checkbox onChange={updateHeader}>{intl.get('import.hasHeader')}</Checkbox>
-                </Form.Item>
-              </Col>
-              <Col span={8}>
-                <Form.Item label={intl.get('import.delimiter')} name="delimiter" required={true}>
-                  <Input placeholder="," onChange={handlePreview} />
-                </Form.Item>
-              </Col>
-            </Row>
+          <Form
+            form={form}
+            layout="horizontal"
+            initialValues={{
+              withHeader,
+              delimiter,
+            }}
+          >
+            {file.name.endsWith('.csv') && (
+              <Row className={styles.configOperation}>
+                <Col span={3}>
+                  <Form.Item name="withHeader" valuePropName="checked">
+                    <Checkbox onChange={updateHeader}>{intl.get('import.hasHeader')}</Checkbox>
+                  </Form.Item>
+                </Col>
+                <Col span={8}>
+                  <Form.Item label={intl.get('import.delimiter')} name="delimiter" required={true}>
+                    <Input placeholder="," onChange={handlePreview} />
+                  </Form.Item>
+                </Col>
+              </Row>
+            )}
             <Form.Item noStyle shouldUpdate={true}>
               {({ getFieldValue }) => {
                 const withHeader = getFieldValue('withHeader');
@@ -127,12 +139,7 @@ const PreviewFileModal = (props: IProps) => {
                 <Button disabled={uploading} onClick={() => handleCancel()}>
                   {intl.get('common.cancel')}
                 </Button>
-                <Button
-                  htmlType="submit"
-                  type="primary"
-                  loading={uploading}
-                  onClick={() => handleConfirm()}
-                >
+                <Button htmlType="submit" type="primary" loading={uploading} onClick={() => handleConfirm()}>
                   {intl.get('common.confirm')}
                 </Button>
               </div>

diff --git a/app/pages/LLMBot/chat.module.less b/app/pages/LLMBot/chat.module.less
@@ -66,7 +66,7 @@
 
   .codeWrapper {
     position: relative;
-    min-width: 360px;
+    min-width: 340px;
 
     >div {
       margin: 10px 0;

diff --git a/app/pages/LLMBot/chat.tsx b/app/pages/LLMBot/chat.tsx
@@ -32,7 +32,7 @@ function Chat() {
     setMessages(newMessages);
     const callback = (res) => {
       if (res.code !== 0) {
-        resetPending(newMessages,res.message);
+        resetPending(newMessages, res.message);
         return;
       }
       if (res.message.done) {
@@ -61,7 +61,7 @@ function Chat() {
         }
         setMessages([...newMessages]);
       } catch (e) {
-        resetPending(newMessages,e.message+"\n response: \n"+JSON.stringify(res.message,null,2));
+        resetPending(newMessages, e.message + '\n response: \n' + JSON.stringify(res.message, null, 2));
       }
     };
     const sendMessages = [
@@ -71,27 +71,38 @@ function Chat() {
         content: item.content.length > 50 ? item.content.trim().slice(0, 50) + '...' : item.content.trim(),
       })),
     ];
-    const systemPrompt = await rootStore.llm.getDocPrompt(currentInput, sendMessages);
-    await ws.runChat({
-      req: {
-        stream: true,
-        messages: [{ role: 'system', content: systemPrompt }, ...sendMessages, { role: 'user', content: currentInput }],
-      },
-      callback,
-    }).catch((e) => {
-      message.error(e.message);
-      resetPending(newMessages,e.message);
+    const systemPrompt = await rootStore.llm.getDocPrompt(currentInput, sendMessages).catch((e) => {
+      resetPending(newMessages, e.message);
     });
+    if (!systemPrompt) return;
+    await ws
+      .runChat({
+        req: {
+          stream: true,
+          messages: [
+            { role: 'system', content: systemPrompt },
+            ...sendMessages,
+            { role: 'user', content: currentInput },
+          ],
+        },
+        callback,
+      })
+      .catch((e) => {
+        message.error(e.message);
+        resetPending(newMessages, e.message);
+      });
   }, 200);
-  const resetPending =  useCallback((messages,msg?:string) => {
+  const resetPending = useCallback((messages, msg?: string) => {
     setPending(false);
-    setMessages(messages.map((item) => {
-      if(item.status !== 'pending') return item;
-      item.status = 'done';
-      item.content = msg ?(`[error] ${msg}`): item.content;
-      return item;
-    }))
-  },[])
+    setMessages(
+      messages.map((item) => {
+        if (item.status !== 'pending') return item;
+        item.status = 'done';
+        item.content = msg ? `[error] ${msg}` : item.content;
+        return item;
+      }),
+    );
+  }, []);
 
   useEffect(() => {
     if (contentRef.current) {
@@ -111,9 +122,11 @@ function Chat() {
     const gqls = message.content.split(/```\w*\n([^`]+)```/);
     return gqls.map((item, index) => {
       if (index % 2 === 0) {
-        return <p key={index}>
-          <pre>{item}</pre>
-        </p>;
+        return (
+          <p key={index}>
+            <pre>{item}</pre>
+          </p>
+        );
       } else {
         item = item.replace(/^(\n|ngql|gql|cypher)/g, '').replace(/\n$/g, '');
         item = item.replace(/\n\n/, '\n');

diff --git a/app/stores/llm.ts b/app/stores/llm.ts
@@ -262,7 +262,12 @@ class LLM {
         },
       })) as any;
       if (res.code === 0) {
-        const url = (res.message.choices[0].message?.content as string)?.split('\n')[0];
+        let url = '';
+        try {
+          url = (res.message.choices[0].message?.content as string)?.split('\n')[0];
+        } catch {
+          throw new Error(JSON.stringify(res.message));
+        }
         const paths = url
           .toLowerCase()
           .replaceAll('，', ',') // chinese comma

diff --git a/server/api/studio/pkg/llm/importjob.go b/server/api/studio/pkg/llm/importjob.go
@@ -148,6 +148,7 @@ func RunFileJob(job *db.LLMJob) {
 		llmJob.SetJobFailed(err)
 		return
 	}
+	llmJob.WriteLogFile(fmt.Sprintf("read file success, file text length: %d", len(text)), "info")
 	blocks, err := llmJob.SplitText(text)
 	if err != nil {
 		llmJob.WriteLogFile(fmt.Sprintf("split text error: %v", err), "error")