add replace spaces flag

anysphere · Sep 30, 2024 · 7f52f3d · 7f52f3d
1 parent b9ac46c
commit 7f52f3d
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/encoding.rs b/src/encoding.rs
@@ -127,7 +127,11 @@ impl Encoding {
         self.core_bpe.encode_ordinary(text)
     }
 
-    pub fn estimate_num_tokens_no_special_tokens_fast(&self, text: &str) -> usize {
+    pub fn estimate_num_tokens_no_special_tokens_fast(&self, text: &str, replace_spaces_with_lower_one_eighth_block: bool = false) -> usize {
+        if replace_spaces_with_lower_one_eighth_block {
+            text = text.replace(" ", "\u{2581}");
+        }
+
         let mut token_count = 0;
         let mut current_token = Vec::new();
         let mut current_token_hash: i64 = 0;