add sentencepiece to pyproject

EleutherAI · Sep 6, 2023 · 4a0a040 · 4a0a040
1 parent 3bbe26c
commit 4a0a040
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -37,8 +37,9 @@ dependencies = [
     # For visualization of results
     "plotly==5.14.1",
     "kaleido==0.2.1",
-    "rich==13.3.5"
-
+    "rich==13.3.5",
+    # neeeded to instantiate tokenizer for encoder-only models
+    "sentencepiece==0.1.99"
 ]
 version = "0.1.1"