Merge pull request #57 from animalnots/dev

1.8.3
animalnots · Sep 8, 2024 · c9b6490 · c9b6490
2 parents 8c4f606 + 547acfe
commit c9b6490
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 32 deletions.
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "better-chatgpt",
   "private": true,
-  "version": "1.8.2",
+  "version": "1.8.3",
   "type": "module",
   "homepage": "./",
   "main": "electron/index.cjs",

diff --git a/src/components/Chat/ChatContent/Message/View/ContentView.tsx b/src/components/Chat/ChatContent/Message/View/ContentView.tsx
@@ -38,6 +38,7 @@ import MarkdownModeButton from './Button/MarkdownModeButton';
 
 import CodeBlock from '../CodeBlock';
 import PopupModal from '@components/PopupModal';
+import { preprocessLaTeX } from '@utils/chat';
 
 const ContentView = memo(
   ({
@@ -144,7 +145,9 @@ const ContentView = memo(
                 p,
               }}
             >
-              {(content[0] as TextContentInterface).text}
+              {inlineLatex
+                ? preprocessLaTeX((content[0] as TextContentInterface).text)
+                : (content[0] as TextContentInterface).text}
             </ReactMarkdown>
           ) : (
             <span className='whitespace-pre-wrap'>

diff --git a/src/utils/chat.ts b/src/utils/chat.ts
@@ -51,3 +51,18 @@ export const downloadMarkdown = (markdown: string, fileName: string) => {
   link.click();
   link.remove();
 };
+
+export const preprocessLaTeX = (content: string) => {
+  // Replace block-level LaTeX delimiters \[ \] with $$ $$
+
+  const blockProcessedContent = content.replace(
+    /\\\[(.*?)\\\]/gs,
+    (_, equation) => `$$${equation}$$`
+  );
+  // Replace inline LaTeX delimiters \( \) with $ $
+  const inlineProcessedContent = blockProcessedContent.replace(
+    /\\\((.*?)\\\)/gs,
+    (_, equation) => `$${equation}$`
+  );
+  return inlineProcessedContent;
+};
diff --git a/src/utils/messageUtils.ts b/src/utils/messageUtils.ts
@@ -1,8 +1,13 @@
-
 import useStore from '@store/store';
 
 import { Tiktoken } from '@dqbd/tiktoken/lite';
-import { isImageContent, isTextContent, MessageInterface, TextContentInterface, TotalTokenUsed } from '@type/chat';
+import {
+  isImageContent,
+  isTextContent,
+  MessageInterface,
+  TextContentInterface,
+  TotalTokenUsed,
+} from '@type/chat';
 import { ModelOptions } from './modelReader';
 const cl100k_base = await import('@dqbd/tiktoken/encoders/cl100k_base.json');
 
@@ -30,7 +35,9 @@ export const getChatGPTEncoding = (
   const serialized = [
     messages
       .map(({ role, content }) => {
-        return `<|im_start|>${role}${roleSep}${(content[0] as TextContentInterface).text}<|im_end|>`;
+        return `<|im_start|>${role}${roleSep}${
+          (content[0] as TextContentInterface).text
+        }<|im_end|>`;
       })
       .join(msgSep),
     `<|im_start|>assistant${roleSep}`,
@@ -52,38 +59,38 @@ export const limitMessageTokens = (
   const limitedMessages: MessageInterface[] = [];
   let tokenCount = 0;
 
-  const isSystemFirstMessage = messages[0]?.role === 'system';
-  let retainSystemMessage = false;
+  // const isSystemFirstMessage = messages[0]?.role === 'system';
+  // let retainSystemMessage = false;
 
-  // Check if the first message is a system message and if it fits within the token limit
-  if (isSystemFirstMessage) {
-    const systemTokenCount = countTokens([messages[0]], model);
-    if (systemTokenCount < limit) {
-      tokenCount += systemTokenCount;
-      retainSystemMessage = true;
-    }
-  }
+  // // Check if the first message is a system message and if it fits within the token limit
+  // if (isSystemFirstMessage) {
+  //   const systemTokenCount = countTokens([messages[0]], model);
+  //   if (systemTokenCount < limit) {
+  //     tokenCount += systemTokenCount;
+  //     retainSystemMessage = true;
+  //   }
+  // }
 
   // Iterate through messages in reverse order, adding them to the limitedMessages array
   // until the token limit is reached (excludes first message)
-  for (let i = messages.length - 1; i >= 1; i--) {
+  for (let i = messages.length - 1; i >= 0; i--) {
     const count = countTokens([messages[i]], model);
     if (count + tokenCount > limit) break;
     tokenCount += count;
     limitedMessages.unshift({ ...messages[i] });
   }
 
-  // Process first message
-  if (retainSystemMessage) {
-    // Insert the system message in the third position from the end
-    limitedMessages.splice(-3, 0, { ...messages[0] });
-  } else if (!isSystemFirstMessage && messages.length > 0) {
-    // Check if the first message (non-system) can fit within the limit
-    const firstMessageTokenCount = countTokens([messages[0]], model);
-    if (firstMessageTokenCount + tokenCount < limit) {
-      limitedMessages.unshift({ ...messages[0] });
-    }
-  }
+  // // Process first message
+  // if (retainSystemMessage) {
+  //   // Insert the system message in the third position from the end
+  //   limitedMessages.splice(-3, 0, { ...messages[0] });
+  // } else if (!isSystemFirstMessage && messages.length > 0) {
+  //   // Check if the first message (non-system) can fit within the limit
+  //   const firstMessageTokenCount = countTokens([messages[0]], model);
+  //   if (firstMessageTokenCount + tokenCount < limit) {
+  //     limitedMessages.unshift({ ...messages[0] });
+  //   }
+  // }
 
   return limitedMessages;
 };
@@ -99,23 +106,30 @@ export const updateTotalTokenUsed = (
   );
 
   // Filter text and image prompts
-  const textPrompts = promptMessages.filter(e => e.content.some(isTextContent));
-  const imgPrompts = promptMessages.filter(e => e.content.some(isImageContent));
+  const textPrompts = promptMessages.filter((e) =>
+    e.content.some(isTextContent)
+  );
+  const imgPrompts = promptMessages.filter((e) =>
+    e.content.some(isImageContent)
+  );
 
   // Count tokens
   const newPromptTokens = countTokens(textPrompts, model);
   const newImageTokens = countTokens(imgPrompts, model);
   const newCompletionTokens = countTokens([completionMessage], model);
 
   // Destructure existing token counts or default to 0
-  const { promptTokens = 0, completionTokens = 0, imageTokens = 0 } =
-    updatedTotalTokenUsed[model] ?? {};
+  const {
+    promptTokens = 0,
+    completionTokens = 0,
+    imageTokens = 0,
+  } = updatedTotalTokenUsed[model] ?? {};
 
   // Update token counts
   updatedTotalTokenUsed[model] = {
     promptTokens: promptTokens + newPromptTokens,
     completionTokens: completionTokens + newCompletionTokens,
-    imageTokens: imageTokens + newImageTokens
+    imageTokens: imageTokens + newImageTokens,
   };
 
   // Set the updated token counts in the store