fix trust_remote_code issur

Signed-off-by: changwangss <[email protected]>

evals/evaluation/lm_evaluation_harness/accuracy.py

@@ -41,7 +41,6 @@ def cli_evaluate(args) -> None:
     eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
     eval_logger.info(f"Verbosity set to {args.verbosity}")
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
-    os.environ["HF_DATASETS_TRUST_REMOTE_CODE"] = "1"
 
     if args.predict_only:
         args.log_samples = True

evals/evaluation/lm_evaluation_harness/lm_eval/models/huggingface.py

@@ -88,7 +88,7 @@ def __init__(
         dtype: Optional[Union[str, torch.dtype]] = "auto",
         batch_size: Optional[Union[int, str]] = 1,
         max_batch_size: Optional[int] = 64,
-        trust_remote_code: Optional[bool] = False,
+        trust_remote_code: Optional[bool] = True,
         use_fast_tokenizer: Optional[bool] = True,
         add_bos_token: Optional[bool] = False,
         prefix_token_id: Optional[int] = None,

tests/test_lm_eval.py

@@ -18,10 +18,17 @@ def test_lm_eval(self):
         user_model = AutoModelForCausalLM.from_pretrained(model_name_or_path)
         tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
         args = LMEvalParser(
-            model="hf", user_model=user_model, tokenizer=tokenizer, tasks="piqa", device="cpu", batch_size=1, limit=5
+            model="hf",
+            user_model=user_model,
+            tokenizer=tokenizer,
+            tasks="lambada_openai",
+            device="cpu",
+            batch_size=1,
+            limit=5,
+            trust_remote_code=True,
         )
         results = evaluate(args)
-        self.assertEqual(results["results"]["piqa"]["acc,none"], 0.6)
+        self.assertEqual(results["results"]["lambada_openai"]["acc,none"], 0.6)
 
 
 if __name__ == "__main__":