Pass kwargs to tokenizer when creating preprocessor from a tokenizer.…

…json
SamanehSaadat · May 17, 2024 · 323abb9 · 323abb9
1 parent 778ccd7
commit 323abb9
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/keras_nlp/src/models/preprocessor.py b/keras_nlp/src/models/preprocessor.py
@@ -181,7 +181,7 @@ def from_preset(
 
         tokenizer = load_serialized_object(preset, TOKENIZER_CONFIG_FILE)
         tokenizer.load_preset_assets(preset)
-        preprocessor = cls(tokenizer=tokenizer)
+        preprocessor = cls(tokenizer=tokenizer, **kwargs)
 
         return preprocessor
 

diff --git a/keras_nlp/src/models/preprocessor_test.py b/keras_nlp/src/models/preprocessor_test.py
@@ -52,6 +52,13 @@ def test_from_preset(self):
             BertMaskedLMPreprocessor,
         )
 
+    @pytest.mark.large
+    def test_from_preset_with_sequence_length(self):
+        preprocessor = BertPreprocessor.from_preset(
+            "bert_tiny_en_uncased", sequence_length=16
+        )
+        self.assertEqual(preprocessor.sequence_length, 16)
+
     @pytest.mark.large
     def test_from_preset_errors(self):
         with self.assertRaises(ValueError):