diff --git a/sentence_transformers/evaluation/NanoBEIREvaluator.py b/sentence_transformers/evaluation/NanoBEIREvaluator.py
index ec29ad094..82907e3ed 100644
--- a/sentence_transformers/evaluation/NanoBEIREvaluator.py
+++ b/sentence_transformers/evaluation/NanoBEIREvaluator.py
@@ -420,6 +420,8 @@ def _load_dataset(self, dataset_name: DatasetNameType, **ir_evaluator_kwargs) ->
         )
 
     def _validate_dataset_names(self):
+        if len(self.dataset_names) == 0:
+            raise ValueError("dataset_names cannot be empty. Use None to evaluate on all datasets.")
         if missing_datasets := [
             dataset_name for dataset_name in self.dataset_names if dataset_name.lower() not in dataset_name_to_id
         ]: