Merge pull request #56 from AGI-Collective/floatingbigcat-patch-1

Update tokenizer.py
CERC-AAI · Sep 25, 2023 · 28e2444 · 28e2444
2 parents d9e0098 + c58f81f
commit 28e2444
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/megatron/tokenizer/tokenizer.py b/megatron/tokenizer/tokenizer.py
@@ -243,7 +243,7 @@ def vocab(self):
     def inv_vocab(self):
         return self.tokenizer.decoder
 
-    def tokenize(self, texts: Union[str, List[str]], pad=True, tensor=True, context_length=2048):
+    def tokenize(self, texts: Union[str, List[str]], pad=True, tensor=True):
         if isinstance(texts, str):
             texts = [texts]
         texts = [whitespace_clean(basic_clean(text)) for text in texts]