Model: Add support for chat_template.json

HuggingFace separated the chat template in the newest transformers versions. Signed-off-by: kingbri <[email protected]>
theroyallab · Nov 11, 2024 · 9530f8c · 9530f8c
1 parent b8700fb
commit 9530f8c
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/backends/exllamav2/model.py b/backends/exllamav2/model.py
@@ -389,6 +389,10 @@ async def find_prompt_template(self, prompt_template_name, model_directory):
         logger.info("Attempting to load a prompt template if present.")
 
         find_template_functions = [
+            lambda: PromptTemplate.from_model_json(
+                pathlib.Path(self.config.model_dir) / "chat_template.json",
+                key="chat_template",
+            ),
             lambda: PromptTemplate.from_model_json(
                 pathlib.Path(self.config.model_dir) / "tokenizer_config.json",
                 key="chat_template",