fix: handle default

ggerganov · Jul 8, 2024 · afd76e6 · afd76e6
1 parent 0699a4c
commit afd76e6
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/src/llama.cpp b/src/llama.cpp
@@ -15468,6 +15468,15 @@ struct llm_tokenizer_bpe {
             case LLAMA_VOCAB_PRE_TYPE_JINA_V2_ZH:
                 regex_exprs = {"\\w+|[^\\w\\s]+"};
                 break;
+            default:
+                // default regex for BPE tokenization pre-processing
+                regex_exprs = {
+                    "[\\p{P}\\$\\+<=>\\^~\\|]+",
+                    "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)",
+                    "\\p{N}+",
+                    "[0-9][0-9][0-9]",
+                };
+                break;
         }
     }