Added regex filter for bbh fewshot

EleutherAI · Nov 18, 2024 · 5b2fcdd · 5b2fcdd
1 parent 62b4364
commit 5b2fcdd
Showing 1 changed file with 14 additions and 0 deletions.
diff --git a/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml b/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml
@@ -15,6 +15,20 @@ generation_kwargs:
     - "\n\n"
   do_sample: false
   temperature: 0.0
+filter_list:
+  - name: remove_whitespace
+    filter:
+      # Filter everything after the first break line
+      - function: "regex"
+        regex_pattern: "^(.*?)(?=\\n|$)"
+      - function: remove_whitespace
+      # function to ignore right white spaces or line breaks
+      - function: "regex"
+        regex_pattern: "^(.*?)\\s*$"
+      # If there are things  between brackets, match those
+      - function: "regex"
+        regex_pattern: "\\([^()]*\\)|[^()]*"
+      - function: take_first
 num_fewshot: 3
 metadata:
   version: 2.0