vocab : add dummy tokens for "no_vocab" type (ggerganov#11231)

* vocab : add dummy tokens for "no_vocab" type ggml-ci * vocab : minor [no ci]
ngxson · Jan 14, 2025 · bbf3e55 · bbf3e55
1 parent c5bf0d1
commit bbf3e55
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
@@ -1356,8 +1356,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
 
             // read vocab size from metadata
             uint32_t n_tokens = 0;
-            if (!ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
-                LLAMA_LOG_WARN("%s: there is no vocab_size in metadata\n", __func__);
+            if (ml.get_key(LLM_KV_VOCAB_SIZE, n_tokens, false)) {
+                LLAMA_LOG_WARN("%s: adding %u dummy tokens\n", __func__, n_tokens);
+                id_to_token.resize(n_tokens);
             }
 
             return;