Prune down LM Eval test time

.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-AWQ.yaml

@@ -0,0 +1,12 @@
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m ybelkada/Mixtral-8x7B-Instruct-v0.1-AWQ -b "auto" -l 1000 -f 5 -t 1
+# Loading model weights took 23.0240 GB
+model_name: "ybelkada/Mixtral-8x7B-Instruct-v0.1-AWQ"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.474
+  - name: "exact_match,flexible-extract"
+    value: 0.498
+limit: 1000
+num_fewshot: 5

.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-GPTQ.yaml

@@ -0,0 +1,12 @@
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m TheBloke/Mixtral-8x7B-Instruct-v0.1-GPTQ -b "auto" -l 1000 -f 5 -t 1
+# Loading model weights took 22.1429 GB
+model_name: "TheBloke/Mixtral-8x7B-Instruct-v0.1-GPTQ"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.570
+  - name: "exact_match,flexible-extract"
+    value: 0.574
+limit: 1000
+num_fewshot: 5

.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml

@@ -1,4 +1,4 @@
-# bash ./run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2-57B-A14B-Instruct -b "auto" -l 250 -f 5 -t 4
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2-57B-A14B-Instruct -b "auto" -l 250 -f 5 -t 4
 model_name: "Qwen/Qwen2-57B-A14B-Instruct"
 tasks:
 - name: "gsm8k"

.buildkite/lm-eval-harness/configs/Qwen2-57B-A14B-Instruct-GPTQ-Int4.yaml

@@ -0,0 +1,11 @@
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4 -b "auto" -l 1000 -f 5 -t 2
+model_name: "Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4"
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.789
+  - name: "exact_match,flexible-extract"
+    value: 0.807
+limit: 1000
+num_fewshot: 5

.buildkite/lm-eval-harness/configs/models-large.txt

@@ -1,5 +1,4 @@
-Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml
-Meta-Llama-3-70B-Instruct.yaml
-Mixtral-8x7B-Instruct-v0.1.yaml
-Qwen2-57B-A14-Instruct.yaml
+Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml
+Mixtral-8x7B-Instruct-v0.1-GPTQ.yaml
+Qwen2-57B-A14B-Instruct-GPTQ-Int4.yaml
 DeepSeek-V2-Lite-Chat.yaml

.buildkite/test-pipeline.yaml

@@ -485,19 +485,17 @@ steps:
   - vllm/
   - tests/weight_loading
   commands:
-    - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
+    - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt 
 
-- label: Weight Loading Multiple GPU Test - Large Models # optional
-  working_dir: "/vllm-workspace/tests"
+- label: LM Eval Large Models # optional
+  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
   num_gpus: 2
-  gpu: a100
-  optional: true
   source_file_dependencies:
-  - vllm/
-  - tests/weight_loading
+  - csrc/
+  - vllm/model_executor/layers/quantization
   commands:
-    - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt 
-
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - bash ./run-tests.sh -c configs/models-large.txt -t 2
 
 ##### multi gpus test #####
 ##### A100 test #####
@@ -514,13 +512,13 @@ steps:
   - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m distributed_2_gpus
   - pytest -v -s -x lora/test_mixtral.py
 
-- label: LM Eval Large Models # optional
+- label: Weight Loading Multiple GPU Test - Large Models # optional
+  working_dir: "/vllm-workspace/tests"
+  num_gpus: 2
   gpu: a100
-  num_gpus: 4
-  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
+  optional: true
   source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
+  - vllm/
+  - tests/weight_loading
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - bash ./run-tests.sh -c configs/models-large.txt -t 4
+    - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt