diff --git a/aicirt/src/bintokens.rs b/aicirt/src/bintokens.rs index 112f985f..e501391b 100644 --- a/aicirt/src/bintokens.rs +++ b/aicirt/src/bintokens.rs @@ -31,12 +31,6 @@ pub fn tokenizers() -> Vec { hf_model: "Xenova/gpt-4", model_ids: "gpt-4", }, - TokenizerInfo { - name: "llama", - description: "used by Llama, CodeLlama, etc.", - hf_model: "codellama/CodeLlama-34b-Instruct-hf", - model_ids: "", - }, TokenizerInfo { name: "llama16", description: "same as llama, with 16 added tokens (used by 13B codellama)", @@ -49,6 +43,12 @@ pub fn tokenizers() -> Vec { hf_model: "codellama/CodeLlama-70b-Instruct-hf", model_ids: "codellama-70b", }, + TokenizerInfo { + name: "llama", + description: "used by Llama, CodeLlama, etc.", + hf_model: "codellama/CodeLlama-34b-Instruct-hf", + model_ids: "", + }, TokenizerInfo { name: "orca", description: "llama",