relax datasets

zenml-io · May 23, 2024 · 1156f4b · 1156f4b
1 parent c33bcfe
commit 1156f4b
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/template/requirements.txt b/template/requirements.txt
@@ -1,11 +1,11 @@
 zenml
 torch>=2.2.0
-datasets<=2.18
+datasets
 transformers
 peft
-bitsandbytes==0.41.3
+bitsandbytes>=0.41.3
 scipy
 evaluate
 rouge_score
 nltk
-accelerate==0.29.2
+accelerate
diff --git a/template/steps/prepare_datasets.py b/template/steps/prepare_datasets.py
@@ -49,8 +49,8 @@ def prepare_data(
     test_dataset = load_dataset(dataset_name, split="test")
 
     datasets_path = Path("datasets")
-    tokenized_train_dataset.save_to_disk(datasets_path / "train")
-    tokenized_val_dataset.save_to_disk(datasets_path / "val")
-    test_dataset.save_to_disk(datasets_path / "test_raw")
+    tokenized_train_dataset.save_to_disk(str((datasets_path / "train").absolute()))
+    tokenized_val_dataset.save_to_disk(str((datasets_path / "val").absolute()))
+    test_dataset.save_to_disk(str((datasets_path / "test_raw").absolute()))
 
     return datasets_path