tokenizer param + misc fixes

huggingface · Dec 20, 2024 · 054f0e5 · 054f0e5
1 parent a1c610d
commit 054f0e5
Showing 1 changed file with 7 additions and 4 deletions.
diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py
@@ -36,7 +36,7 @@
     GenerativeResponse,
     LoglikelihoodResponse,
 )
-from lighteval.models.utils import _get_dtype, _simplify_name
+from lighteval.models.utils import _get_dtype, _get_model_sha, _simplify_name
 from lighteval.tasks.requests import (
     GreedyUntilRequest,
     LoglikelihoodRequest,
@@ -89,6 +89,9 @@ class VLLMModelConfig:
     subfolder: Optional[str] = None
     temperature: float = 0.6  # will be used for multi sampling tasks, for tasks requiring no sampling, this will be ignored and set to 0.
 
+    def get_model_sha(self):
+        return _get_model_sha(repo_id=self.pretrained, revision=self.revision)
+
 
 class VLLMModel(LightevalModel):
     def __init__(
@@ -113,10 +116,10 @@ def __init__(
         self.multichoice_continuations_start_space = config.multichoice_continuations_start_space
 
         self.model_name = _simplify_name(config.pretrained)
-        self.model_sha = ""  # config.get_model_sha()
+        self.model_sha = config.get_model_sha()
         self.precision = _get_dtype(config.dtype, config=self._config)
 
-        self.model_info = ModelInfo(model_name=self.model_name, model_sha=self.model_sha)
+        self.model_info = ModelInfo(model_name=self.model_name, model_sha=self.model_sha, model_dtype=config.dtype)
         self.pairwise_tokenization = config.pairwise_tokenization
 
     @property
@@ -191,7 +194,7 @@ def _create_auto_tokenizer(self, config: VLLMModelConfig, env_config: EnvConfig)
             config.pretrained,
             tokenizer_mode="auto",
             trust_remote_code=config.trust_remote_code,
-            tokenizer_revision=config.revision,
+            revision=config.revision + (f"/{config.subfolder}" if config.subfolder is not None else ""),
         )
         tokenizer.pad_token = tokenizer.eos_token
         return tokenizer