🐛 Bug: Fix the bug in calculating the number of PDF tokens.

yym68686 · Oct 8, 2024 · 4361551 · 4361551
1 parent 8224202
commit 4361551
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/setup.py b/setup.py
@@ -4,7 +4,7 @@
 
 setup(
     name="modelmerge",
-    version="0.11.51",
+    version="0.11.52",
     description="modelmerge is a multi-large language model API aggregator.",
     long_description=Path.open(Path("README.md"), encoding="utf-8").read(),
     long_description_content_type="text/markdown",

diff --git a/src/ModelMerge/utils/scripts.py b/src/ModelMerge/utils/scripts.py
@@ -9,7 +9,7 @@ def get_encode_text(text, model_name):
     tiktoken.get_encoding("cl100k_base")
     model_name = "gpt-3.5-turbo"
     encoding = tiktoken.encoding_for_model(model_name)
-    encode_text = encoding.encode(text)
+    encode_text = encoding.encode(text, disallowed_special=())
     return encoding, encode_text
 
 def get_text_token_len(text, model_name):