remove random select in tooleval

OpenBMB · Nov 17, 2023 · fbad8a0 · fbad8a0
1 parent b062b2c
commit fbad8a0
Show file tree

Hide file tree

Showing 5 changed files with 7 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -577,7 +577,7 @@ python eval_pass_rate.py \
     --reference_model ${CANDIDATE_MODEL} \
     --test_ids ../../data/test_ids/ \
     --max_eval_threads 20 \
-    --evaluate_times 4
+    --evaluate_times 7
 
 ```
 The result files will be stored under the ${SAVE_PATH}.
@@ -600,7 +600,7 @@ python eval_preference.py \
     --pass_rate_result_path ${PASS_TARE_PATH} \
     --max_eval_threads 20 \
     --use_pass_rate true \
-    --evaluate_times 4
+    --evaluate_times 7
 ```
 The result files will be stored under the ${SAVE_PATH}.
 

diff --git a/README_ZH.md b/README_ZH.md
@@ -585,7 +585,7 @@ python eval_pass_rate.py \
     --reference_model ${CANDIDATE_MODEL} \
     --test_ids ../../data/test_query_ids/ \
     --max_eval_threads 20 \
-    --evaluate_times 4
+    --evaluate_times 7
 
 ```
 
@@ -609,7 +609,7 @@ python eval_preference.py \
     --pass_rate_result_path ${PASS_TARE_PATH} \
     --max_eval_threads 20 \
     --use_pass_rate true \
-    --evaluate_times 4
+    --evaluate_times 7
 ```
 
 结果文件会被存储至${SAVE_PATH}中。

diff --git a/toolbench/tooleval/eval_pass_rate.py b/toolbench/tooleval/eval_pass_rate.py
@@ -170,11 +170,8 @@ def compute_pass_rate(query_id, example):
         write_results(filename, reference_model, label_cnt)
         pass_rate = 0
         for query_id in label_cnt:
-            if label_cnt[query_id]["failed"] < label_cnt[query_id]["passed"]:
+            if label_cnt[query_id]["failed"] <= label_cnt[query_id]["passed"]:
                 pass_rate += 1
-            elif label_cnt[query_id]["failed"] == label_cnt[query_id]["passed"]:
-                if random.random() < 0.5:
-                    pass_rate += 1
         pass_rate /= len(label_cnt)
         print(f"Test set: {test_set}. Model: {reference_model}. Pass rate: {str(pass_rate)}")
 

diff --git a/toolbench/tooleval/run_pass_rate.sh b/toolbench/tooleval/run_pass_rate.sh
@@ -9,4 +9,4 @@ python eval_pass_rate.py \
     --reference_model ${CANDIDATE_MODEL} \
     --test_ids ../../data/test_query_ids/ \
     --max_eval_threads 20 \
-    --evaluate_times 4
+    --evaluate_times 7
diff --git a/toolbench/tooleval/run_preference.sh b/toolbench/tooleval/run_preference.sh
@@ -14,4 +14,4 @@ python eval_preference.py \
     --pass_rate_result_path ${PASS_TARE_PATH} \
     --max_eval_threads 20 \
     --use_pass_rate true \
-    --evaluate_times 4
+    --evaluate_times 7