fix tokenizer auto-detection (order)

microsoft · Feb 23, 2024 · e801c75 · e801c75
1 parent f997ad9
commit e801c75
Showing 1 changed file with 6 additions and 6 deletions.
diff --git a/aicirt/src/bintokens.rs b/aicirt/src/bintokens.rs
@@ -31,12 +31,6 @@ pub fn tokenizers() -> Vec<TokenizerInfo> {
             hf_model: "Xenova/gpt-4",
             model_ids: "gpt-4",
         },
-        TokenizerInfo {
-            name: "llama",
-            description: "used by Llama, CodeLlama, etc.",
-            hf_model: "codellama/CodeLlama-34b-Instruct-hf",
-            model_ids: "",
-        },
         TokenizerInfo {
             name: "llama16",
             description: "same as llama, with 16 added tokens (used by 13B codellama)",
@@ -49,6 +43,12 @@ pub fn tokenizers() -> Vec<TokenizerInfo> {
             hf_model: "codellama/CodeLlama-70b-Instruct-hf",
             model_ids: "codellama-70b",
         },
+        TokenizerInfo {
+            name: "llama",
+            description: "used by Llama, CodeLlama, etc.",
+            hf_model: "codellama/CodeLlama-34b-Instruct-hf",
+            model_ids: "",
+        },
         TokenizerInfo {
             name: "orca",
             description: "llama",