006: Updating spacy version

mudes/app/mudes_app.py

@@ -1,6 +1,6 @@
 from spacy.lang.en import English
 from spacy.lang.xx import MultiLanguage
-
+from spacy.tokenizer import Tokenizer
 from mudes.algo.mudes_model import MUDESModel
 from mudes.algo.predict import predict_spans
 from mudes.algo.preprocess import contiguous_ranges
@@ -53,7 +53,7 @@ def predict_tokens(self, text: str, language: str = "en"):
         else:
             nlp = MultiLanguage()
 
-        tokenizer = nlp.Defaults.create_tokenizer(nlp)
+        tokenizer = Tokenizer(nlp.vocab)
         tokens = tokenizer(text)
         output_tokens = []
         for token in tokens:

requirements.txt

@@ -13,4 +13,4 @@ wandb
 sentencepiece
 onnxruntime
 seqeval
-spacy
+spacy==3.5.1

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="mudes",
-    version="0.3.0",
+    version="0.4.0",
     author="Tharindu Ranasinghe",
     author_email="[email protected]",
     description="Toxic Spans Prediction",
@@ -36,6 +36,6 @@
         "sentencepiece",
         "onnxruntime",
         "seqeval",
-        "spacy"
+        "spacy==3.5.1"
     ],
 )