add <|fim_pad|> for extra_stop_tokens of qwen

smallcloudai · Dec 3, 2024 · bfabbeb · bfabbeb
1 parent 55e6af1
commit bfabbeb
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/known_models.rs b/src/known_models.rs
@@ -210,7 +210,7 @@ pub const KNOWN_MODELS: &str = r####"
                     "fim_suffix": "<|fim_suffix|>",
                     "fim_middle": "<|fim_middle|>",
                     "eot": "<|endoftext|>",
-                    "extra_stop_tokens": ["<|repo_name|>", "<|file_sep|>"],
+                    "extra_stop_tokens": ["<|repo_name|>", "<|file_sep|>", "<|fim_pad|>"],
                     "context_format": "qwen2.5",
                     "rag_ratio": 0.5
                 }