Merge pull request #8 from HanXudong/dataset

Dataset
Libr-AI · Jul 14, 2022 · 9c58c74 · 9c58c74
2 parents 4b82649 + 28ec75e
commit 9c58c74
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 2 deletions.
diff --git a/fairlib/datasets/utils/download.py b/fairlib/datasets/utils/download.py
@@ -2,7 +2,7 @@
 import requests
 from tqdm.auto import tqdm
 
-def download(url: str, dest_folder: str, chunk_size: int = 1024*8):
+def download(url: str, dest_folder: str, chunk_size: int = 10*1024*1024):
     if not os.path.exists(dest_folder):
         os.makedirs(dest_folder)  # create folder if it does not exist
 

diff --git a/requirements.txt b/requirements.txt
@@ -9,3 +9,4 @@ seaborn==0.11.2
 matplotlib==3.5.0
 pickle5==0.0.12
 transformers==4.11.3
+sacremoses==0.0.53
diff --git a/setup.py b/setup.py
@@ -27,6 +27,7 @@
         "matplotlib",
         "pickle5",
         "transformers",
+        "sacremoses",
     ]
 
 
@@ -35,7 +36,7 @@
 if __name__ == '__main__':
     setup(
         name='fairlib',
-        version="0.0.6",
+        version="0.0.7",
         author="Xudong Han",
         author_email="[email protected]",
         description='Unified framework for assessing and improving fairness.',