fixed parsing token vocabularies for gemma and gpt-sw3 models

dottxt-ai · Mar 25, 2024 · c744e25 · c744e25
1 parent f7cafe5
commit c744e25
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/outlines/fsm/regex.py b/outlines/fsm/regex.py
@@ -719,7 +719,7 @@ def create_fsm_index_end_to_end(
 
 
 re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
-re_replacement_seq = re.compile(r"^�+$")
+re_replacement_seq = re.compile(r"^▁*�+$")
 
 
 # Copied from transformers.models.gpt2.tokenization_gpt2.bytes_to_unicode