fix deepseek remote code

vllm-project · Aug 6, 2024 · 5a6b8e1 · 5a6b8e1
1 parent 360bd67
commit 5a6b8e1
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml b/.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
@@ -9,3 +9,4 @@ tasks:
     value: 0.664
 limit: 1000
 num_fewshot: 5
+trust_remote_code: True
diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py
@@ -23,9 +23,15 @@
 
 
 def launch_lm_eval(eval_config):
+    try:
+        trust_remote_code = eval_config['trust_remote_code']
+    except:
+        trust_remote_code = False
+
     model_args = f"pretrained={eval_config['model_name']}," \
                  f"tensor_parallel_size={TP_SIZE}," \
-                 f"add_bos_token=true"
+                 f"add_bos_token=true," \
+                 f"trust_remote_code={trust_remote_code}"
 
     results = lm_eval.simple_evaluate(
         model="vllm",